<!DOCTYPE html>
<!-- saved from url=(0127)https://docs.google.com/document/d/e/2PACX-1vScbVZC19jesKXrwmfOG3qSs7WF49wvqCRA0Vm3AElOofsN9g4t9v5AnQC_CX6aWgRzMdkAHhZI04IH/pub -->
<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"><title>WebPerfWG open meeting - A/B testing</title><link rel="shortcut icon" href="https://ssl.gstatic.com/docs/documents/images/kix-favicon7.ico"><meta name="referrer" content="strict-origin-when-cross-origin"><style type="text/css" nonce="">
      @import url("https://fonts.googleapis.com/css?family=Google+Sans");
      @import url("https://fonts.googleapis.com/css?family=Roboto");

      body {
        font-family: Roboto, arial, sans, sans-serif;
        margin: 0;
      }

      iframe {
        border: 0;
        frameborder: 0;
        height: 100%;
        width: 100%;
      }

      #header {
        align-items: center;
        background: white;
        border-bottom: 1px #ccc solid;
        display: flex;
        height: 60px;
        justify-content: space-between;
        position: fixed;
        top: 0;
        width: 100%;
        z-index: 100;
      }

      #header #title {
        font-family: 'Google Sans';
        font-size: large;
        margin: auto 0 auto 20px;
        overflow: hidden;
        text-overflow: ellipsis;
        white-space: nowrap;
        width: 70%;
      }

      #header #interval {
        margin: auto 25px auto 0;
        font-family: Roboto;
        font-size: small;;
      }

      #footer {
        background: #f0f0f0;
        border-bottom: 1px #ccc solid;
        bottom: 0;
        font-family: Roboto;
        font-size: small;
        padding: 10px 10px;
        position: fixed;
        text-align: center;
        width: 100%;
      }

      #contents {
        padding: 100px 20% 50px 20%;
      }

      @media only screen and (max-device-width: 800px) {
        #header {
          border-bottom-width: 5px;
          height: auto;
          display: block;
        }

        #header #title {
          font-size: 3em;
          margin: auto 0 auto 20px;
          width: 90%;
        }

        #header #interval {
          font-size: 1.5em;
          margin: 10px 0 auto 25px;
        }

        #contents {
          padding: 150px 5% 80px;
        }

        #footer {
          font-size: 2em;
        }
      }

      .dash {
        padding: 0 6px;
      }
    </style></head><body><div id="header"><div id="title">WebPerfWG open meeting - A/B testing</div><div id="interval"><span></span></div></div><div id="contents"><style type="text/css">.lst-kix_pua8k7j8se9e-6>li:before{content:"\0025cf  "}.lst-kix_rarm0ndq7v2i-0>li:before{content:"\0025cf  "}.lst-kix_rarm0ndq7v2i-1>li:before{content:"\0025cb  "}.lst-kix_ka67hqwwdw6x-8>li:before{content:"\0025a0  "}.lst-kix_ka67hqwwdw6x-7>li:before{content:"\0025cb  "}.lst-kix_pua8k7j8se9e-3>li:before{content:"\0025cf  "}.lst-kix_pua8k7j8se9e-7>li:before{content:"\0025cb  "}.lst-kix_rarm0ndq7v2i-5>li:before{content:"\0025a0  "}.lst-kix_ka67hqwwdw6x-4>li:before{content:"\0025cb  "}.lst-kix_ka67hqwwdw6x-5>li:before{content:"\0025a0  "}.lst-kix_pua8k7j8se9e-4>li:before{content:"\0025cb  "}.lst-kix_rarm0ndq7v2i-6>li:before{content:"\0025cf  "}.lst-kix_ka67hqwwdw6x-6>li:before{content:"\0025cf  "}.lst-kix_pua8k7j8se9e-5>li:before{content:"\0025a0  "}.lst-kix_rarm0ndq7v2i-8>li:before{content:"\0025a0  "}ul.lst-kix_q3zi7brx38zu-6{list-style-type:none}ul.lst-kix_q3zi7brx38zu-5{list-style-type:none}.lst-kix_rarm0ndq7v2i-7>li:before{content:"\0025cb  "}ul.lst-kix_q3zi7brx38zu-4{list-style-type:none}ul.lst-kix_q3zi7brx38zu-3{list-style-type:none}ul.lst-kix_q3zi7brx38zu-2{list-style-type:none}ul.lst-kix_q3zi7brx38zu-1{list-style-type:none}ul.lst-kix_q3zi7brx38zu-0{list-style-type:none}.lst-kix_pua8k7j8se9e-8>li:before{content:"\0025a0  "}ul.lst-kix_q3zi7brx38zu-8{list-style-type:none}ul.lst-kix_q3zi7brx38zu-7{list-style-type:none}.lst-kix_21pe4vbi3tu7-5>li:before{content:"\0025a0  "}ul.lst-kix_cl521vqmc48c-8{list-style-type:none}ul.lst-kix_cl521vqmc48c-7{list-style-type:none}.lst-kix_u3e7zrarnh6k-8>li:before{content:"\0025a0  "}ul.lst-kix_cl521vqmc48c-6{list-style-type:none}.lst-kix_21pe4vbi3tu7-4>li:before{content:"\0025cb  "}.lst-kix_21pe4vbi3tu7-8>li:before{content:"\0025a0  "}ul.lst-kix_cl521vqmc48c-1{list-style-type:none}ul.lst-kix_cl521vqmc48c-0{list-style-type:none}ul.lst-kix_cl521vqmc48c-5{list-style-type:none}.lst-kix_21pe4vbi3tu7-7>li:before{content:"\0025cb  "}ul.lst-kix_cl521vqmc48c-4{list-style-type:none}ul.lst-kix_cl521vqmc48c-3{list-style-type:none}.lst-kix_21pe4vbi3tu7-6>li:before{content:"\0025cf  "}ul.lst-kix_cl521vqmc48c-2{list-style-type:none}.lst-kix_21pe4vbi3tu7-0>li:before{content:"\0025cf  "}ul.lst-kix_5z3ihgj0q4tq-3{list-style-type:none}.lst-kix_21pe4vbi3tu7-1>li:before{content:"\0025cb  "}ul.lst-kix_5z3ihgj0q4tq-4{list-style-type:none}ul.lst-kix_5z3ihgj0q4tq-1{list-style-type:none}ul.lst-kix_5z3ihgj0q4tq-2{list-style-type:none}ul.lst-kix_5z3ihgj0q4tq-7{list-style-type:none}.lst-kix_21pe4vbi3tu7-3>li:before{content:"\0025cf  "}ul.lst-kix_5z3ihgj0q4tq-8{list-style-type:none}ul.lst-kix_5z3ihgj0q4tq-5{list-style-type:none}.lst-kix_21pe4vbi3tu7-2>li:before{content:"\0025a0  "}ul.lst-kix_5z3ihgj0q4tq-6{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-3{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-8{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-4{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-1{list-style-type:none}.lst-kix_ljmgazj753yk-8>li:before{content:"\0025a0  "}ul.lst-kix_u3e7zrarnh6k-2{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-7{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-4{list-style-type:none}.lst-kix_ljmgazj753yk-7>li:before{content:"\0025cb  "}ul.lst-kix_u3e7zrarnh6k-8{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-5{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-5{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-6{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-6{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-7{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-0{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-1{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-2{list-style-type:none}ul.lst-kix_ka67hqwwdw6x-3{list-style-type:none}ul.lst-kix_u3e7zrarnh6k-0{list-style-type:none}ul.lst-kix_pua8k7j8se9e-5{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-7{list-style-type:none}ul.lst-kix_pua8k7j8se9e-6{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-8{list-style-type:none}ul.lst-kix_pua8k7j8se9e-7{list-style-type:none}ul.lst-kix_pua8k7j8se9e-8{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-3{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-4{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-5{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-6{list-style-type:none}.lst-kix_cl521vqmc48c-6>li:before{content:"\0025cf  "}ul.lst-kix_17xu3lqzjo8t-0{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-1{list-style-type:none}ul.lst-kix_pua8k7j8se9e-0{list-style-type:none}ul.lst-kix_17xu3lqzjo8t-2{list-style-type:none}.lst-kix_cl521vqmc48c-8>li:before{content:"\0025a0  "}ul.lst-kix_pua8k7j8se9e-1{list-style-type:none}ul.lst-kix_pua8k7j8se9e-2{list-style-type:none}.lst-kix_cl521vqmc48c-7>li:before{content:"\0025cb  "}ul.lst-kix_pua8k7j8se9e-3{list-style-type:none}ul.lst-kix_pua8k7j8se9e-4{list-style-type:none}.lst-kix_u3e7zrarnh6k-7>li:before{content:"\0025cb  "}.lst-kix_u3e7zrarnh6k-6>li:before{content:"\0025cf  "}.lst-kix_u3e7zrarnh6k-5>li:before{content:"\0025a0  "}.lst-kix_u3e7zrarnh6k-3>li:before{content:"\0025cf  "}.lst-kix_u3e7zrarnh6k-2>li:before{content:"\0025a0  "}.lst-kix_u3e7zrarnh6k-4>li:before{content:"\0025cb  "}.lst-kix_ka67hqwwdw6x-1>li:before{content:"\0025cb  "}.lst-kix_u3e7zrarnh6k-0>li:before{content:"\0025cf  "}.lst-kix_u3e7zrarnh6k-1>li:before{content:"\0025cb  "}.lst-kix_ka67hqwwdw6x-3>li:before{content:"\0025cf  "}.lst-kix_ka67hqwwdw6x-0>li:before{content:"\0025cf  "}.lst-kix_ka67hqwwdw6x-2>li:before{content:"\0025a0  "}.lst-kix_17xu3lqzjo8t-4>li:before{content:"\0025cb  "}.lst-kix_5z3ihgj0q4tq-0>li:before{content:"\0025cf  "}.lst-kix_17xu3lqzjo8t-1>li:before{content:"\0025cb  "}.lst-kix_17xu3lqzjo8t-5>li:before{content:"\0025a0  "}.lst-kix_5z3ihgj0q4tq-1>li:before{content:"\0025cb  "}.lst-kix_5z3ihgj0q4tq-2>li:before{content:"\0025a0  "}.lst-kix_5z3ihgj0q4tq-4>li:before{content:"\0025cb  "}.lst-kix_17xu3lqzjo8t-2>li:before{content:"\0025a0  "}.lst-kix_17xu3lqzjo8t-3>li:before{content:"\0025cf  "}.lst-kix_5z3ihgj0q4tq-3>li:before{content:"\0025cf  "}ul.lst-kix_21pe4vbi3tu7-0{list-style-type:none}.lst-kix_17xu3lqzjo8t-0>li:before{content:"\0025cf  "}.lst-kix_cl521vqmc48c-2>li:before{content:"\0025a0  "}ul.lst-kix_ljmgazj753yk-8{list-style-type:none}ul.lst-kix_ljmgazj753yk-7{list-style-type:none}ul.lst-kix_ljmgazj753yk-4{list-style-type:none}ul.lst-kix_ljmgazj753yk-3{list-style-type:none}.lst-kix_cl521vqmc48c-1>li:before{content:"\0025cb  "}.lst-kix_cl521vqmc48c-5>li:before{content:"\0025a0  "}ul.lst-kix_ljmgazj753yk-6{list-style-type:none}ul.lst-kix_ljmgazj753yk-5{list-style-type:none}.lst-kix_cl521vqmc48c-4>li:before{content:"\0025cb  "}.lst-kix_cl521vqmc48c-3>li:before{content:"\0025cf  "}ul.lst-kix_21pe4vbi3tu7-1{list-style-type:none}.lst-kix_ljmgazj753yk-2>li:before{content:"\0025a0  "}ul.lst-kix_21pe4vbi3tu7-2{list-style-type:none}ul.lst-kix_21pe4vbi3tu7-3{list-style-type:none}.lst-kix_5z3ihgj0q4tq-6>li:before{content:"\0025cf  "}ul.lst-kix_21pe4vbi3tu7-4{list-style-type:none}.lst-kix_5z3ihgj0q4tq-8>li:before{content:"\0025a0  "}ul.lst-kix_21pe4vbi3tu7-5{list-style-type:none}.lst-kix_ljmgazj753yk-3>li:before{content:"\0025cf  "}.lst-kix_ljmgazj753yk-4>li:before{content:"\0025cb  "}ul.lst-kix_21pe4vbi3tu7-6{list-style-type:none}ul.lst-kix_21pe4vbi3tu7-7{list-style-type:none}.lst-kix_5z3ihgj0q4tq-5>li:before{content:"\0025a0  "}ul.lst-kix_21pe4vbi3tu7-8{list-style-type:none}.lst-kix_17xu3lqzjo8t-8>li:before{content:"\0025a0  "}ul.lst-kix_ljmgazj753yk-0{list-style-type:none}.lst-kix_ljmgazj753yk-5>li:before{content:"\0025a0  "}.lst-kix_ljmgazj753yk-6>li:before{content:"\0025cf  "}ul.lst-kix_ljmgazj753yk-2{list-style-type:none}ul.lst-kix_ljmgazj753yk-1{list-style-type:none}.lst-kix_cl521vqmc48c-0>li:before{content:"\0025cf  "}.lst-kix_17xu3lqzjo8t-6>li:before{content:"\0025cf  "}.lst-kix_5z3ihgj0q4tq-7>li:before{content:"\0025cb  "}.lst-kix_17xu3lqzjo8t-7>li:before{content:"\0025cb  "}ul.lst-kix_5z3ihgj0q4tq-0{list-style-type:none}.lst-kix_q3zi7brx38zu-7>li:before{content:"\0025cb  "}.lst-kix_q3zi7brx38zu-8>li:before{content:"\0025a0  "}.lst-kix_q3zi7brx38zu-3>li:before{content:"\0025cf  "}.lst-kix_ljmgazj753yk-1>li:before{content:"\0025cb  "}.lst-kix_ljmgazj753yk-0>li:before{content:"\0025cf  "}.lst-kix_q3zi7brx38zu-2>li:before{content:"\0025a0  "}.lst-kix_q3zi7brx38zu-6>li:before{content:"\0025cf  "}.lst-kix_q3zi7brx38zu-5>li:before{content:"\0025a0  "}.lst-kix_q3zi7brx38zu-4>li:before{content:"\0025cb  "}ul.lst-kix_rarm0ndq7v2i-7{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-6{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-8{list-style-type:none}.lst-kix_q3zi7brx38zu-1>li:before{content:"\0025cb  "}li.li-bullet-0:before{margin-left:-18pt;white-space:nowrap;display:inline-block;min-width:18pt}.lst-kix_q3zi7brx38zu-0>li:before{content:"\0025cf  "}ul.lst-kix_rarm0ndq7v2i-1{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-0{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-3{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-2{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-5{list-style-type:none}ul.lst-kix_rarm0ndq7v2i-4{list-style-type:none}.lst-kix_pua8k7j8se9e-2>li:before{content:"\0025a0  "}.lst-kix_rarm0ndq7v2i-4>li:before{content:"\0025cb  "}.lst-kix_pua8k7j8se9e-0>li:before{content:"\0025cf  "}.lst-kix_rarm0ndq7v2i-2>li:before{content:"\0025a0  "}.lst-kix_rarm0ndq7v2i-3>li:before{content:"\0025cf  "}.lst-kix_pua8k7j8se9e-1>li:before{content:"\0025cb  "}ol{margin:0;padding:0}table td,table th{padding:0}.c16{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;background-color:#999999;border-left-style:solid;border-bottom-width:1pt;width:220.5pt;border-top-color:#000000;border-bottom-style:solid}.c27{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;background-color:#f3f3f3;border-left-style:solid;border-bottom-width:1pt;width:63pt;border-top-color:#000000;border-bottom-style:solid}.c28{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;background-color:#b6d7a8;border-left-style:solid;border-bottom-width:1pt;width:220.5pt;border-top-color:#000000;border-bottom-style:solid}.c9{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;background-color:#b6d7a8;border-left-style:solid;border-bottom-width:1pt;width:63pt;border-top-color:#000000;border-bottom-style:solid}.c24{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;background-color:#999999;border-left-style:solid;border-bottom-width:1pt;width:63pt;border-top-color:#000000;border-bottom-style:solid}.c26{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;border-left-style:solid;border-bottom-width:1pt;width:183.8pt;border-top-color:#000000;border-bottom-style:solid}.c30{border-right-style:solid;padding:5pt 5pt 5pt 5pt;border-bottom-color:#000000;border-top-width:1pt;border-right-width:1pt;border-left-color:#000000;vertical-align:top;border-right-color:#000000;border-left-width:1pt;border-top-style:solid;border-left-style:solid;border-bottom-width:1pt;width:220.5pt;border-top-color:#000000;border-bottom-style:solid}.c3{margin-left:72pt;padding-top:0pt;padding-left:0pt;padding-bottom:0pt;line-height:1.15;orphans:2;widows:2;text-align:left}.c2{margin-left:36pt;padding-top:0pt;padding-left:0pt;padding-bottom:0pt;line-height:1.15;orphans:2;widows:2;text-align:left}.c1{color:#000000;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:11pt;font-family:"Arial";font-style:normal}.c21{color:#000000;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:16pt;font-family:"Arial";font-style:normal}.c18{padding-top:20pt;padding-bottom:6pt;line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}.c10{padding-top:18pt;padding-bottom:6pt;line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}.c7{color:#000000;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:20pt;font-family:"Arial";font-style:normal}.c12{padding-top:14pt;padding-bottom:4pt;line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}.c15{color:#666666;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:12pt;font-family:"Arial";font-style:normal}.c13{color:#434343;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:14pt;font-family:"Arial";font-style:normal}.c19{padding-top:16pt;padding-bottom:4pt;line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}.c17{color:#000000;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:12pt;font-family:"Arial";font-style:normal}.c36{color:#000000;font-weight:400;text-decoration:none;vertical-align:baseline;font-size:11pt;font-family:"Arial";font-style:italic}.c0{color:#000000;font-weight:700;text-decoration:none;vertical-align:baseline;font-size:9pt;font-family:"Arial";font-style:normal}.c20{color:#000000;text-decoration:none;vertical-align:baseline;font-size:11pt;font-family:"Arial";font-style:normal}.c8{padding-top:0pt;padding-bottom:0pt;line-height:1.15;orphans:2;widows:2;text-align:left}.c34{font-weight:400;vertical-align:baseline;font-size:11pt;font-family:"Arial";font-style:normal}.c33{padding-top:0pt;padding-bottom:0pt;line-height:1.0;text-align:left}.c23{padding-top:0pt;padding-bottom:0pt;line-height:1.0;text-align:center}.c4{text-decoration-skip-ink:none;-webkit-text-decoration-skip:none;color:#1155cc;text-decoration:underline}.c32{border-spacing:0;border-collapse:collapse;margin-right:auto}.c22{background-color:#ffffff;max-width:468pt;padding:72pt 72pt 72pt 72pt}.c5{padding:0;margin:0}.c31{margin-left:108pt;padding-left:0pt}.c6{color:inherit;text-decoration:inherit}.c29{background-color:#b6d7a8}.c25{height:11pt}.c14{height:0pt}.c37{font-style:italic}.c35{background-color:#999999}.c11{font-weight:700}.title{padding-top:0pt;color:#000000;font-size:26pt;padding-bottom:3pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}.subtitle{padding-top:0pt;color:#666666;font-size:15pt;padding-bottom:16pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}li{color:#000000;font-size:11pt;font-family:"Arial"}p{margin:0;color:#000000;font-size:11pt;font-family:"Arial"}h1{padding-top:20pt;color:#000000;font-size:20pt;padding-bottom:6pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}h2{padding-top:18pt;color:#000000;font-size:16pt;padding-bottom:6pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}h3{padding-top:16pt;color:#434343;font-size:14pt;padding-bottom:4pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}h4{padding-top:14pt;color:#666666;font-size:12pt;padding-bottom:4pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}h5{padding-top:12pt;color:#666666;font-size:11pt;padding-bottom:4pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;orphans:2;widows:2;text-align:left}h6{padding-top:12pt;color:#666666;font-size:11pt;padding-bottom:4pt;font-family:"Arial";line-height:1.15;page-break-after:avoid;font-style:italic;orphans:2;widows:2;text-align:left}</style><div class="c22"><p class="c8"><span class="c17">The Web Performance WG invites the broader web performance community to discuss A/B testing and ways we can collectively work to make it faster! The goal of this open meeting is to explore the problem space, try to reach common understandings on the benefits and shortcomings of the current approaches, and brainstorm a brighter future and how we can get there. </span></p><p class="c8 c25"><span class="c17"></span></p><p class="c8"><span class="c1">We plan to have representatives from across the web performance and A/B testing community, including:</span></p><ul class="c5 lst-kix_ka67hqwwdw6x-0 start"><li class="c2 li-bullet-0"><span class="c1">WebPerf WG members</span></li><li class="c2 li-bullet-0"><span class="c1">A/B testing providers</span></li><li class="c2 li-bullet-0"><span class="c1">Web performance consultants</span></li><li class="c2 li-bullet-0"><span class="c1">CDNs</span></li><li class="c2 li-bullet-0"><span>Content platforms</span></li></ul><h1 class="c18" id="h.3cczxhsp2bjh"><span class="c7">Logistics</span></h1><h2 class="c10" id="h.n78mt3418jzx"><span class="c21">When</span></h2><p class="c8"><span class="c11 c37">Thursday February 4th, 12-2:30pm PST</span><span class="c1">&nbsp;(9-11:30pm CET, 8-10:30pm GMT, 5-7:30am JST)</span></p><h2 class="c10" id="h.icl64nez1bci"><span class="c21">Where</span></h2><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=http://meet.google.com/bqq-bouy-ghz&amp;sa=D&amp;source=editors&amp;ust=1613054543283000&amp;usg=AOvVaw2MA3x7w3CyccyTV6KC0P0O">Online</a></span><span class="c1">!</span></p><p class="c8"><span class="c1">As always with WG calls, the meeting will be recorded and recordings as well as minutes will be posted online.</span></p><h2 class="c10" id="h.2iao0kr3dulw"><span class="c21">Attendees</span></h2><p class="c8"><span class="c1">To register, simply add your name, email and employer (if relevant) below. </span></p><ul class="c5 lst-kix_ljmgazj753yk-0 start"><li class="c2 li-bullet-0"><span>Yoav Weiss - </span><span class="c4"><a class="c6" href="mailto:yoav@yoav.ws">yoav@yoav.ws</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Nicolás Peña Moreno - </span><span class="c4"><a class="c6" href="mailto:npm@google.com">npm@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Michael Hood - </span><span class="c4"><a class="c6" href="mailto:michaelhood@gmail.com">michaelhood@gmail.com</a></span></li><li class="c2 li-bullet-0"><span>Naina Raisinghani - </span><span class="c4"><a class="c6" href="mailto:nainar@google.com">nainar@google.com</a></span><span class="c1">, - Google</span></li><li class="c2 li-bullet-0"><span>Bing Chen - </span><span class="c4"><a class="c6" href="mailto:bingc@google.com">bingc@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span class="c1">Dimitris Dimitropoulos - ddimitrop@google.com - Google</span></li><li class="c2 li-bullet-0"><span>Nathan Tate - </span><span class="c4"><a class="c6" href="mailto:nathantate@google.com">nathantate@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Kristofer Baxter - </span><span class="c4"><a class="c6" href="mailto:kbax@google.com">kbax@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Ishan Anand - </span><span class="c4"><a class="c6" href="mailto:ishan@moovweb.com">ishan@moovweb.com</a></span><span class="c1">&nbsp;- Moovweb</span></li><li class="c2 li-bullet-0"><span>Patrick Meenan - </span><span class="c4"><a class="c6" href="mailto:pmeenan@webpagetest.org">pmeenan@webpagetest.org</a></span><span class="c1">&nbsp;- Catchpoint</span></li><li class="c2 li-bullet-0"><span>Addy Osmani - </span><span class="c4"><a class="c6" href="mailto:addyo@google.com">addyo@google.com</a></span><span class="c1">&nbsp;- Google </span></li><li class="c2 li-bullet-0"><span>Kenji Baheux - </span><span class="c4"><a class="c6" href="mailto:kenjibaheux@google.com">kenjibaheux@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Noam Helfman - </span><span class="c4"><a class="c6" href="mailto:noamh@microsoft.com">noamh@microsoft.com</a></span><span class="c1">&nbsp;- Microsoft</span></li><li class="c2 li-bullet-0"><span>Peter Perlepes - </span><span class="c4"><a class="c6" href="mailto:p.perlepes@gmail.com">p.perlepes@gmail.com</a></span><span class="c1">&nbsp;- trivago</span></li><li class="c2 li-bullet-0"><span>Carine Bournez- </span><span class="c4"><a class="c6" href="mailto:carine@w3.org">carine@w3.org</a></span><span class="c1">&nbsp;- W3C</span></li><li class="c2 li-bullet-0"><span>Palash barua </span><span class="c4"><a class="c6" href="mailto:-palash.barua@aamra.com.bd">- palash.barua@aamra.com.bd</a></span><span class="c1">&nbsp;- aamra technologies limited</span></li><li class="c2 li-bullet-0"><span class="c1">Andy Davies, hello@andydavies.me - Independent Consultant</span></li><li class="c2 li-bullet-0"><span>Keith Freund, </span><span class="c4"><a class="c6" href="mailto:brainwavescode@gmail.com">brainwavescode@gmail.com</a></span><span>&nbsp;- </span><span class="c1">Freelance Web Developer</span></li><li class="c2 li-bullet-0"><span>Tim Kadlec, </span><span class="c4"><a class="c6" href="mailto:tim@timkadlec.com">tim@timkadlec.com</a></span><span class="c1">&nbsp;- Catchpoint</span></li><li class="c2 li-bullet-0"><span>Ryan Townsend, </span><span class="c4"><a class="c6" href="mailto:ryan@ryantownsend.co.uk">ryan@ryantownsend.co.uk</a></span><span class="c1">&nbsp;– SHIFT Commerce</span></li><li class="c2 li-bullet-0"><span>Nirbhab Barat, </span><span class="c4"><a class="c6" href="mailto:nirbhab@lambdatest.com">nirbhab@lambdatest.com</a></span><span class="c1">&nbsp;- LambdaTest.com </span></li><li class="c2 li-bullet-0"><span>Mayank Bhola </span><span class="c4"><a class="c6" href="mailto:mayank@lambdatest.com">mayank@lambdatest.com</a></span><span class="c1">&nbsp;- LambdaTest.com</span></li><li class="c2 li-bullet-0"><span>Srivishnu Ayyagari </span><span class="c4"><a class="c6" href="mailto:srivishnua@lambdatest.com">srivishnua@lambdatest.com</a></span><span class="c1">&nbsp;- Lambdatest.com</span></li><li class="c2 li-bullet-0"><span>Melissa Mitchell - </span><span class="c4"><a class="c6" href="mailto:ellemitchell@google.com">ellemitchell@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Deeksha Agarwal- </span><span class="c4"><a class="c6" href="mailto:deekshaa@lambdatest.com">deekshaa@lambdatest.com</a></span><span class="c1">- LambdaTest</span></li><li class="c2 li-bullet-0"><span>Nic Jansma - </span><span class="c4"><a class="c6" href="mailto:njansma@akamai.com">njansma@akamai.com</a></span><span class="c1">&nbsp;- Akamai</span></li><li class="c2 li-bullet-0"><span>Benjamin De Kosnik - </span><span class="c4"><a class="c6" href="mailto:bdekoz@mozilla.com">bdekoz@mozilla.com</a></span><span class="c1">&nbsp;- Mozilla</span></li><li class="c2 li-bullet-0"><span>Johan Aldor - </span><span class="c4"><a class="c6" href="mailto:jaldor@akamai.com">jaldor@akamai.com</a></span><span class="c1">&nbsp;- Akamai</span></li><li class="c2 li-bullet-0"><span>Tim Dresser - </span><span class="c4"><a class="c6" href="mailto:tdresser@google.com">tdresser@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Dharani Govindan - </span><span class="c4"><a class="c6" href="mailto:dharani@google.com">dharani@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Shubhie Panicker </span><span class="c4"><a class="c6" href="mailto:panicker@google.com">panicker@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Philip Walton - </span><span class="c4"><a class="c6" href="mailto:philipwalton@google.com">philipwalton@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Naidu Venkat- </span><span class="c4"><a class="c6" href="mailto:naiduvr@lambdatest.com">naiduvr@lambdatest.com</a></span><span class="c1">- Lambdatest</span></li><li class="c2 li-bullet-0"><span>Harry Roberts – </span><span class="c4"><a class="c6" href="mailto:csswizardry@gmail.com">csswizardry@gmail.com</a></span><span class="c1">&nbsp;– N/A</span></li><li class="c2 li-bullet-0"><span>Andrew Galloni - </span><span class="c4"><a class="c6" href="mailto:andrew@cloudflare.com">andrew@cloudflare.com</a></span><span class="c1">&nbsp;- Cloudflare</span></li><li class="c2 li-bullet-0"><span>Colin Bendell - </span><span class="c4"><a class="c6" href="mailto:colin.bendell@shopify.com">colin.bendell@shopify.com</a></span><span class="c1">&nbsp;- Shopify</span></li><li class="c2 li-bullet-0"><span>CP Clermont - </span><span class="c4"><a class="c6" href="mailto:cp.clermont@shopify.com">cp.clermont@shopify.com</a></span><span class="c1">&nbsp;- Shopify</span></li><li class="c2 li-bullet-0"><span>Warren Maresca - </span><span class="c4"><a class="c6" href="mailto:warrengm@google.com">warrengm@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span class="c1">Ilya Grigorik — ilya@grigorik.com</span></li><li class="c2 li-bullet-0"><span class="c1">Leo Lamprecht – leo@vercel.com – Vercel</span></li><li class="c2 li-bullet-0"><span>Bel Curcio - </span><span class="c4"><a class="c6" href="mailto:bel@vercel.com">bel@vercel.com</a></span><span class="c1">&nbsp;- Vercel</span></li><li class="c2 li-bullet-0"><span>Eric Smyth - </span><span class="c4"><a class="c6" href="mailto:esmyth@mozilla.com">esmyth@mozilla.com</a></span><span class="c1">&nbsp;- Mozilla</span></li><li class="c2 li-bullet-0"><span>Lucas Pardue - </span><span class="c4"><a class="c6" href="mailto:lucas@cloudflare.com">lucas@cloudflare.com</a></span><span class="c1">&nbsp;- Cloudflare</span></li><li class="c2 li-bullet-0"><span>Hong Xiao - </span><span class="c4"><a class="c6" href="mailto:jeffhongxiao@gmail.com">jeffhongxiao@gmail.com</a></span><span class="c1">&nbsp;- N/A</span></li><li class="c2 li-bullet-0"><span>Michelle Vu - </span><span class="c4"><a class="c6" href="mailto:michellevu@pinterest.com">michellevu@pinterest.com</a></span><span class="c1">&nbsp;- Pinterest</span></li><li class="c2 li-bullet-0"><span>Scott Connelly- </span><span class="c4"><a class="c6" href="mailto:sconnelly@sitespect.com">sconnelly@sitespect.com</a></span><span class="c1">&nbsp;-SiteSpect </span></li><li class="c2 li-bullet-0"><span>Greeshma Yellareddy - </span><span class="c4"><a class="c6" href="mailto:greeshma@optimizely.com">greeshma</a></span><span class="c4 c34">y@gmail.com - Optimizely</span></li><li class="c2 li-bullet-0"><span>Shalom Volchok - </span><span class="c4"><a class="c6" href="mailto:shalom@outsmartly.com">shalom@outsmartly.com</a></span><span class="c1">&nbsp;- Outsmartly</span></li><li class="c2 li-bullet-0"><span>Jay Phelps - </span><span class="c4"><a class="c6" href="mailto:jay@outsmartly.com">jay@outsmartly.com</a></span><span class="c1">&nbsp;- Outsmartly</span></li><li class="c2 li-bullet-0"><span>Sam Fischgrund - </span><span class="c4"><a class="c6" href="mailto:sfischgrund@google.com">sfischgrund@google.com</a></span><span class="c1">&nbsp;- Google</span></li><li class="c2 li-bullet-0"><span>Aly Cabral - </span><span class="c4"><a class="c6" href="mailto:aly@cloudflare.com">aly@cloudflare.com</a></span><span class="c1">&nbsp;- Cloudflare</span></li><li class="c2 li-bullet-0"><span class="c1">Glen Maddern - glen@cloudflare.com - Cloudflare</span></li><li class="c2 li-bullet-0"><span>Mark Moeckel - </span><span class="c4"><a class="c6" href="mailto:mark.moeckel@dominos.com">mark.moeckel@dominos.com</a></span><span class="c1">&nbsp;- Domino’s</span></li><li class="c2 li-bullet-0"><span>Patrick Hamann - </span><span class="c4"><a class="c6" href="mailto:patrick@fastly.com">patrick@fastly.com</a></span><span class="c1">&nbsp;- Fastly</span></li><li class="c2 li-bullet-0"><span>Aditya Punjani - </span><span class="c4"><a class="c6" href="mailto:aditya.punjani@airbnb.com">aditya.punjani@airbnb.com</a></span><span class="c1">&nbsp;- Airbnb</span></li><li class="c2 li-bullet-0"><span>Aman Nanner - </span><span class="c4"><a class="c6" href="mailto:ananner@akamai.com">ananner@akamai.com</a></span><span class="c1">&nbsp;- Akamai</span></li><li class="c2 li-bullet-0"><span>Rachel Wilkins Patel - </span><span class="c4"><a class="c6" href="mailto:rachel@bridgeandswitch.com">rachel@bridgeandswitch.com</a></span><span class="c1">&nbsp;- n/a</span></li><li class="c2 li-bullet-0"><span>James Russell - </span><span class="c4"><a class="c6" href="mailto:james.russell@dominos.com">james.russell@dominos.com</a></span><span class="c1">&nbsp;- Domino’s</span></li><li class="c2 li-bullet-0"><span>Utkarsh Goel - </span><span class="c4"><a class="c6" href="mailto:ugoel@akamai.com">ugoel@akamai.com</a></span><span class="c1">&nbsp;- Akamai</span></li><li class="c2 li-bullet-0"><span>Katie Hempenius - </span><span class="c4"><a class="c6" href="mailto:khempenius@google.com">khempenius@google.com</a></span></li><li class="c2 li-bullet-0"><span class="c1">Michal Mocny - mmocny@google.com</span></li><li class="c2 li-bullet-0"><span class="c1">Marcel Duran - marcelduran@google.com - Google</span></li><li class="c2 li-bullet-0"><span>Dave Pifke - </span><span class="c4"><a class="c6" href="mailto:dpifke@wikimedia.org">dpifke@wikimedia.org</a></span><span class="c1">&nbsp;- Wikimedia Foundation</span></li><li class="c2 li-bullet-0"><span>Paul Bernier - </span><span class="c4"><a class="c6" href="mailto:pbernier@sitespect.com">pbernier@sitespect.com</a></span><span class="c1">&nbsp;- SiteSpect</span></li></ul><h2 class="c10" id="h.5qi7hwe8qqy9"><span class="c21">Reading material</span></h2><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/document/d/1tUePot24bqpZmW31jmGa1FHMJ12YkXXubD3S1tnih0M/edit&amp;sa=D&amp;source=editors&amp;ust=1613054543314000&amp;usg=AOvVaw07JUSNaDGcTxVgHoTxpeNl">Real-Life Performance and Client-Side A/B Testing Challenges</a></span><span class="c1">&nbsp;</span></p><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/document/d/1SVRl4qwr5sVUsAITkrvMJPTMSuJatNRFfUKgx4P1rH8/edit%23&amp;sa=D&amp;source=editors&amp;ust=1613054543315000&amp;usg=AOvVaw37tterEmaPQRwxf58qP23K">Client side A/B testing outside of the client</a></span><span class="c1">&nbsp;</span></p><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/11mHDr3u4py1zOlq5Fd1ub0335rXYfxjp8wMfXF0--E0/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543315000&amp;usg=AOvVaw2cyE_F9EHXQuiX8Nn28-3j">Client-side A/B Testing and Performance (a match made somewhere other than heaven)</a></span></p><h1 class="c18" id="h.qh2v613m4y49"><span class="c7">Agenda</span></h1><p class="c8"><span class="c1">Times in PST</span></p><p class="c8 c25"><span class="c36"></span></p><a id="t.5b0b536525f8fbe1e5116247bdec85636558d47d"></a><a id="t.0"></a><table class="c32"><tbody><tr class="c14"><td class="c24" colspan="1" rowspan="1"><p class="c23"><span class="c0">Timeslot</span></p></td><td class="c26 c35" colspan="1" rowspan="1"><p class="c33"><span class="c1">Subject</span></p></td><td class="c16" colspan="1" rowspan="1"><p class="c33"><span class="c1">POC</span></p></td></tr><tr class="c14"><td class="c27" colspan="1" rowspan="1"><p class="c33"><span class="c0">12:00-12:10</span></p></td><td class="c26" colspan="1" rowspan="1"><p class="c33"><span class="c11">Intro</span></p></td><td class="c30" colspan="1" rowspan="1"><p class="c33"><span class="c1">Yoav</span></p></td></tr><tr class="c14"><td class="c27" colspan="1" rowspan="1"><p class="c23"><span class="c0">12:10-12:30</span></p></td><td class="c26" colspan="1" rowspan="1"><p class="c8"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/11mHDr3u4py1zOlq5Fd1ub0335rXYfxjp8wMfXF0--E0/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543321000&amp;usg=AOvVaw26H4eby6QglRQXj_7TQuEk">Client-Side A/B Testing &amp; Performance</a></span><span class="c11"><br><br></span><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/15AA_4_aJJxH3qckVUu6FSRzKotCSapEteSM8PJKEiSE/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543322000&amp;usg=AOvVaw0L_0hmf2o5Fr4unJGU8g3p">Real life problems with client-side A/B testing</a></span></p></td><td class="c30" colspan="1" rowspan="1"><p class="c8"><span>Tim Kadlec</span><span class="c1">&nbsp;- CatchPoint, </span></p><p class="c8"><span>Melissa Mitchell</span><span class="c1">&nbsp;- Google</span></p></td></tr><tr class="c14"><td class="c27" colspan="1" rowspan="1"><p class="c23"><span class="c0">12:30-13:00</span></p></td><td class="c26" colspan="1" rowspan="1"><p class="c8"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1QwEP4m-EneSYkW1QtD8SN7eAyjxsaobo6XIXMU3tDcU/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543325000&amp;usg=AOvVaw0Q1z8vu2JTORjvBUDVEUmP">How client-side A/B testing works</a></span><span class="c11">&nbsp;- </span><span class="c20 c11">Optimizely</span></p><p class="c8"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1M414_RHF--1IVyYb6V_9wxCsa-yRO5_HGsIvkY__sas/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543326000&amp;usg=AOvVaw00ydeqGbricmbhP2tSU5hP">Google Optimize</a></span></p></td><td class="c30" colspan="1" rowspan="1"><p class="c8"><span>Greeshma Yellareddy</span><span class="c1">&nbsp;- Optimizely,</span></p><p class="c8"><span>Dimitris Dimitropoulos</span><span>&nbsp;- </span><span>Google Optimize</span></p></td></tr><tr class="c14"><td class="c9" colspan="1" rowspan="1"><p class="c23"><span class="c0">13:00~13:05</span></p></td><td class="c26 c29" colspan="1" rowspan="1"><p class="c8"><span class="c1">Break</span></p></td><td class="c28" colspan="1" rowspan="1"><p class="c8 c25"><span class="c1"></span></p></td></tr><tr class="c14"><td class="c27" colspan="1" rowspan="1"><p class="c23"><span class="c0">13:05-13:15</span></p></td><td class="c26" colspan="1" rowspan="1"><p class="c8"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1rmVcHJtfJhFvASGfsbPjwguW7kP9D8Le0YW6zx_vk-0/edit%23slide%3Did.p&amp;sa=D&amp;source=editors&amp;ust=1613054543331000&amp;usg=AOvVaw2vkeTdD2TlWFBzSTK9avCF">Client-side A/B testing outside the client??</a></span></p></td><td class="c30" colspan="1" rowspan="1"><p class="c8"><span class="c1">Yoav Weiss - Google</span></p></td></tr><tr class="c14"><td class="c27" colspan="1" rowspan="1"><p class="c23"><span class="c0">13:15-14:30</span></p></td><td class="c26" colspan="1" rowspan="1"><p class="c8"><span class="c20 c11">Brainstorming</span></p></td><td class="c30" colspan="1" rowspan="1"><p class="c8 c25"><span class="c1"></span></p></td></tr></tbody></table><p class="c8 c25"><span class="c1"></span></p><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://www.w3.org/Consortium/cepc/&amp;sa=D&amp;source=editors&amp;ust=1613054543334000&amp;usg=AOvVaw0N43kUiQLCZOk0en0bmjoz">W3C’s Code of Ethics and Professional Conduct</a></span></p><h1 class="c18" id="h.fv3e9glh0tjg"><span class="c7">Conclusions</span></h1><ul class="c5 lst-kix_rarm0ndq7v2i-0 start"><li class="c2 li-bullet-0"><span class="c1">A/B testing providers need better ways to load their scripts</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-1 start"><li class="c3 li-bullet-0"><span class="c1">The library+experiment code would benefit from “render blocking but not parser blocking” semantics, potentially with a developer defined timeout.</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-2 start"><li class="c8 c31 li-bullet-0"><span class="c1">That would enable to improve performance and eliminate SPOF and need for anti-flicker snippets.</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-1"><li class="c3 li-bullet-0"><span class="c1">Analytics code would benefit from a simpler way to load the code at idle time</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-0"><li class="c2 li-bullet-0"><span class="c1">Browser support for a standard-track transformation language could benefit the industry</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-1 start"><li class="c3 li-bullet-0"><span class="c1">A/B testing providers seemed willing to support such a language</span></li><li class="c3 li-bullet-0"><span class="c1">At the same time, there was some skepticism about the potential performance benefits</span></li></ul><ul class="c5 lst-kix_rarm0ndq7v2i-0"><li class="c2 li-bullet-0"><span class="c1">Web shared libraries could reduce the amount of code each A/B tested page loads</span></li><li class="c2 li-bullet-0"><span class="c1">There’s appetite for native support for an “experiment variant” primitive, over today’s use of cookies for this purpose, for both convenience as well as caching reasons.</span></li><li class="c2 li-bullet-0"><span class="c1">The definition of First Paint could take anti-flicker snippets into account and surface cases where UX is degraded by them (e.g. by turning the page blank)</span></li><li class="c2 li-bullet-0"><span class="c1">May be interesting to explore avoiding the paint of specific elements on the page, rather than blocking the full render.</span></li><li class="c2 li-bullet-0"><span class="c1">There’s interest in being able to surface the impact of A/B testing to the people making the decisions. There may be room for an A/B testing User Timing convention that can help on that front.</span></li><li class="c2 li-bullet-0"><span class="c1">The A/B testing providers present (Optimizely and Google Optimize) have an API that would enable CDNs to e.g. remove the A/B testing script in case no experiments are running.</span></li></ul><p class="c8 c25"><span class="c1"></span></p><h1 class="c18" id="h.fdtju8ydw8xf"><span class="c7">Recordings</span></h1><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://youtu.be/2XIVI1jcztY&amp;sa=D&amp;source=editors&amp;ust=1613054543337000&amp;usg=AOvVaw07G-ZitxhQhLgDGGVlmCBc">Part 1</a></span><span class="c1">&nbsp;- Talks</span></p><p class="c8"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://youtu.be/b8fjyAf9veY&amp;sa=D&amp;source=editors&amp;ust=1613054543338000&amp;usg=AOvVaw0dtXvPVtlwyzlfSZpMfV8k">Part 2</a></span><span>&nbsp;- Brainstorming</span></p><h1 class="c18" id="h.dfx6wtbjkkkt"><span class="c7">Minutes</span></h1><h2 class="c10" id="h.1h7oyfr7pc71"><span class="c21">Intro</span></h2><ul class="c5 lst-kix_u3e7zrarnh6k-0 start"><li class="c2 li-bullet-0"><span class="c1">&nbsp;<b>Yoav</b>: A lot of misunderstanding between web performance and A/B testing communities</span></li><li class="c2 li-bullet-0"><span class="c1">… We have common goals of making sites faster and better for users</span></li><li class="c2 li-bullet-0"><span class="c1">… Try to figure out a solution</span></li><li class="c2 li-bullet-0"><span class="c1">… Today we have talks from Tim and Melissa on real-life problems we see today</span></li><li class="c2 li-bullet-0"><span class="c1">… Greeshma and Dimitris talk about how A/B testing works today</span></li><li class="c2 li-bullet-0"><span class="c1">… Will have a short proposal as well</span></li><li class="c2 li-bullet-0"><span class="c1">… This meeting is covered by W3C Code of Professional Conduct</span></li></ul><h2 class="c10" id="h.i4gvirn4pz31"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/11mHDr3u4py1zOlq5Fd1ub0335rXYfxjp8wMfXF0--E0/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543341000&amp;usg=AOvVaw14z6rJXi33F10S3bybD4HQ">Client-Side A/B Testing &amp; Performance - Tim Kadlec</a></span></h2><ul class="c5 lst-kix_pua8k7j8se9e-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Tim</b>: Work on WPT at Catchpoint!</span></li><li class="c2 li-bullet-0"><span class="c1">… Prior to that was performance consulting, and this is from things I was seeing</span></li><li class="c2 li-bullet-0"><span class="c1">… Client-side A/B testing works by providing some JS to browser, which may manipulate DOM or CSS to apply A/B testing</span></li><li class="c2 li-bullet-0"><span class="c1">… As you would expect, any time we’re doing this, we want to try to avoid shifts or flickering of content</span></li><li class="c2 li-bullet-0"><span class="c1">… Challenge for A/B providers is finding a way to combat this: how do we get experiments down to browser as efficiently as possible to ensure shifting/flickering doesn’t occur</span></li><li class="c2 li-bullet-0"><span class="c1">… Easiest option is to drop in a synchronous script</span></li><li class="c2 li-bullet-0"><span class="c1">… Gets higher priority on the network, blocks page until experiments are applied</span></li><li class="c2 li-bullet-0"><span class="c1">… From a performance perspective, we’ve now moved that 3P into the critical path</span></li><li class="c2 li-bullet-0"><span class="c1">… e.g. on example site w/ Optimizely the script causes a block of 2.3+seconds</span></li><li class="c2 li-bullet-0"><span class="c1">… HTML parsing is completely paused during that time</span></li><li class="c2 li-bullet-0"><span class="c1">… There’s predictable correlation between engagement and conversion rates and first paints, and this slows it down significantly </span></li><li class="c2 li-bullet-0"><span class="c1">… A blocking script also introduces a SPOF failure risk</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: I would like to know how often that happens</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Tim</b>: Yes we can try to look into that</span></li><li class="c2 li-bullet-0"><span>… Recommended reading on </span><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://csswizardry.com/2019/05/self-host-your-static-assets/&amp;sa=D&amp;source=editors&amp;ust=1613054543344000&amp;usg=AOvVaw11U7-qmhGT-qpllq2Jrdvj">self-hosting vs. third-party domains</a></span></li><li class="c2 li-bullet-0"><span class="c1">… Alternative approach is to use an async script, to mitigate SPOF risk</span></li><li class="c2 li-bullet-0"><span class="c1">… But now the page could be displayed before those experiments are applied, risking shifts and flicker</span></li><li class="c2 li-bullet-0"><span class="c1">… Typically when we see this, it’s paired with some CSS by setting opacity:0 or something similar to avoid flicker</span></li><li class="c2 li-bullet-0"><span class="c1">… Introduces its own type of risk, where CSS hides content and JavaScript never loads, page remains blank</span></li><li class="c2 li-bullet-0"><span class="c1">… Google Optimize default snippet sets 4 seconds timeout for JavaScript to load</span></li><li class="c2 li-bullet-0"><span class="c1">… If it doesn’t come in within that window, then page will be displayed and experiments won’t run</span></li><li class="c2 li-bullet-0"><span class="c1">… Google Optimizer, Adobe Target, Visual Web Optimizer have different default timeouts of 4/3/2.5s</span></li><li class="c2 li-bullet-0"><span class="c1">… These timeouts may seem high. 4s is a lot</span></li><li class="c2 li-bullet-0"><span class="c1">… We’re potentially pushing out render for that many of seconds</span></li><li class="c2 li-bullet-0"><span class="c1">… But timeouts have to be high because the async script may come late (low priority)</span></li><li class="c2 li-bullet-0"><span class="c1">… Example from Andy Davies where VWO has low priority, in second loading phase, contents with other body content (and is uncacheable)</span></li><li class="c2 li-bullet-0"><span class="c1">… Page is hidden for two seconds before that script arrives and executes</span></li><li class="c2 li-bullet-0"><span class="c1">… Can be hard to catch with RUM as FCP may fire early because opacity:0 is contentful</span></li><li class="c2 li-bullet-0"><span class="c1">… Some examples where RUM looks great and only when we look at WPT that you can see a problem</span></li><li class="c2 li-bullet-0"><span class="c1">… Gets worse when using a tag manager for doing experiments</span></li><li class="c2 li-bullet-0"><span class="c1">… One audit shown had the 86th request on the page be the request that applies the experiments and renders the page</span></li><li class="c2 li-bullet-0"><span class="c1">… They were reaching Google Optimize timeout of 5 seconds about 30% of the time</span></li><li class="c2 li-bullet-0"><span>… Andy wrote a great </span><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://andydavies.me/blog/2020/11/16/the-case-against-anti-flicker-snippets/&amp;sa=D&amp;source=editors&amp;ust=1613054543348000&amp;usg=AOvVaw0JWg_gBf6sVGoR6sWQeTz8">post on anti-flickr snippets</a></span><span class="c1">&nbsp;</span></li><li class="c2 li-bullet-0"><span class="c1">… A single snippet can include many events, pages and experiments</span></li><li class="c2 li-bullet-0"><span class="c1">… Experiments may have a tendency to linger on and may be inefficiently written</span></li><li class="c2 li-bullet-0"><span class="c1">… We’ll often see large JS payload for the experiment, resulting in Long Tasks</span></li><li class="c2 li-bullet-0"><span class="c1">… Observing the DOM can be expensive</span></li><li class="c2 li-bullet-0"><span class="c1">… A/B testing is the perfect marriage of third-party risk and JavaScript bottlenecks</span></li><li class="c2 li-bullet-0"><span class="c1">… Why people use it? Some data is only on the client, can be used to workaround dev cycles </span></li><li class="c2 li-bullet-0"><span class="c1">… Perceived cost that Client-side A/B is cheaper than perf</span></li><li class="c2 li-bullet-0"><span class="c1">… But has to be offset by the cost of lost conversions because of performance degradation</span></li><li class="c2 li-bullet-0"><span class="c1">… Can also be simpler to setup</span></li></ul><h3 class="c19" id="h.lj2r68tpe6ew"><span class="c13">Chat conversation</span></h3><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: Does anyone have data about how often SPOF is actually failing?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Michal</b>: Does the anti-flicker timeout result in population bias for the test?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-2 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: performance (user experience) itself introduces population bias into A/B tests. I think you could consider the timeout as resulting in a "trimmed" population. If the statistical accuracy is that deeply important to the administrator of the test, it's one of many things they'd need to consider that current tools don't natively take into account.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: Timeouts won't be evenly distributed across the population, though. &nbsp;Users on slow connections will be under-represented in the test.</span></li><li class="c3 li-bullet-0"><span><b>Nathan Tate</b>: The problem is a little more complicated than that. Users on slow connections may even only timeout some of the times, which means you have a group of users who are getting an inconsistent experience, sometimes seeing the original, and sometimes seeing the variant</span></li></ul><h2 class="c10" id="h.1779vnfa56w0"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/15AA_4_aJJxH3qckVUu6FSRzKotCSapEteSM8PJKEiSE/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543352000&amp;usg=AOvVaw0nu1spo9O-i2jcfKZ22GDS">Real life problems with client-side A/B testing</a></span><span class="c21">&nbsp;- Melissa Mitchell</span></h2><ul class="c5 lst-kix_21pe4vbi3tu7-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Melissa</b>: Web Ecosystem Consultant from Google focused on Core Web Vitals</span></li><li class="c2 li-bullet-0"><span class="c1">… A/B testing almost always comes up, one of the struggles they’re working around in meeting CWV</span></li><li class="c2 li-bullet-0"><span class="c1">… Example of zales.com using Maximiser AMP+PWA</span></li><li class="c2 li-bullet-0"><span class="c1">… FCP increases by 4 seconds (likely hitting timeout)</span></li><li class="c2 li-bullet-0"><span class="c1">… LCP delayed by a second as well</span></li><li class="c2 li-bullet-0"><span class="c1">… Additional shift of 0.17 just from this A/B testing library</span></li><li class="c2 li-bullet-0"><span class="c1">… Another example for NPR, best-case scenario from out of the box A/B testing</span></li><li class="c2 li-bullet-0"><span class="c1">… React SPA (not AMP) with their A/B testing they see about a second impact on their pages</span></li><li class="c2 li-bullet-0"><span class="c1">… LCP is last piece for meeting CWV, and they’re right on the edge</span></li><li class="c2 li-bullet-0"><span class="c1">… That A/B impact is the factor in meeting CWV</span></li><li class="c2 li-bullet-0"><span class="c1">… Challenges: Primary A/B test creators are typically not in technical roles, using WYSIWYG editors to bypass internal bureaucracy</span></li><li class="c2 li-bullet-0"><span style="overflow: hidden;<b> display</b>: inline-block;<b> margin</b>: 0.00px 0.00px;<b> border</b>: 0.00px solid #000000;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);<b> width</b>: 624.00px;<b> height</b>: 352.00px;"><img alt="" src="./WebPerfWG open meeting - A_B testing_files/Screen Shot 2021-02-08 at 15.52.00.png" style="width: 624.00px;<b> height</b>: 352.00px; margin-left: 0.00px; margin-top: 0.00px;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></li><li class="c2 li-bullet-0"><span class="c1">… Universal Truth: Users are never going to use software in the way developers intend</span></li><li class="c2 li-bullet-0"><span class="c1">… Corollary: Your users may not be who you think and intend them to be</span></li><li class="c2 li-bullet-0"><span class="c1">… Business Use Cases for Client-Side Testing:</span></li><li class="c2 li-bullet-0"><span style="overflow: hidden;<b> display</b>: inline-block;<b> margin</b>: 0.00px 0.00px;<b> border</b>: 0.00px solid #000000;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);<b> width</b>: 624.00px;<b> height</b>: 352.00px;"><img alt="" src="./WebPerfWG open meeting - A_B testing_files/Screen Shot 2021-02-08 at 15.52.30.png" style="width: 624.00px;<b> height</b>: 352.00px; margin-left: 0.00px; margin-top: 0.00px;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></li><li class="c2 li-bullet-0"><span class="c1">… Increase conversions/revenue, reduce workload on marketing teams (not often technical),</span></li><li class="c2 li-bullet-0"><span class="c1">… Reduce outsourced engineering costs</span></li><li class="c2 li-bullet-0"><span class="c1">… Allows tests independent of release cycles (many companies slow with cycles, every 2 weeks or every 2 months). &nbsp;Businesses use A/B testing to adjust to this, rather than put chages in through code deployments.</span></li><li class="c2 li-bullet-0"><span class="c1">… Successful tests live until development work is done</span></li><li class="c2 li-bullet-0"><span class="c1">… Avoid excess design work for tests, can be extra burden in some processes</span></li><li class="c2 li-bullet-0"><span class="c1">… Can double as personalization library</span></li><li class="c2 li-bullet-0"><span class="c1">… Avoid tech debt, 70% of tests are unsuccessful or inconclusive, so not adding them to the code base permanently helps</span></li><li class="c2 li-bullet-0"><span class="c1">… A/B testing is solving these pain points</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Questions from chat:</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Alex</b>: Is solution to serve content A/B from first-party and eliminate all third-party delay</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Melissa</b>: Need some JavaScript to decide is person is in group A or B, someone else may be able to talk about whether 1P could help</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Alex</b>: One thing I noticed from scraping websites is they’d often respond with content A or B if there was no cookie. I wonder why we’re not pushing people towards that model</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Melissa</b>: If you have to have page A and B written, then engineering resources have to write those pages. &nbsp;Teams doing these tests are not engineers, and may not get engineering resources allocated fast enough.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Andy</b>: Many people use platforms that just don’t support it, i.e. using ecommerce platforms or publishing platforms that don’t have any concepts of experimentation built into them.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Melissa</b>: Kenji’s point that it has to be free or cheap enough, there’s always a cost-benefit</span></li></ul><h3 class="c19" id="h.snhp3gbiqy06"><span class="c13">Chat conversation</span></h3><ul class="c5 lst-kix_cl521vqmc48c-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Alex</b>: I may be naive, but isn't the obvious solution to all of this to just serve either content A or content B from the first party and eliminate all this third party timeout delay?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Andy</b>: Many sites use ecommerce / publish platforms that don't support it</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma</b>: Also typically, there can be anywhere between 1-5 experiments per page, each with a different targeting conditions. And further, with SPA, navigation means you will have to apply experiments without loading from server side</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji</b>: Probably also increases the IT bureaucracy factor / implementation burden, which makes it less appealing than the current 3P + client-side approach.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya</b>: I can't think of any platforms that support A/B testing as a first-class thing. For those that have dedicated dev teams, this is plausible but not easy; most of the web doesn't have access to developer time and resources. Plus other bits that Melissa and Tim mentioned: speed, access to client-side data, WYSIWG, etc.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andy</b>: Sitespect is the only CMS I know of that supports testing as a first class thing</span></li><li class="c3 li-bullet-0"><span class="c1">Michael Hood :What Greeshma said is the biggest fundamental issue from my POV, the proliferation of SPA / client-side hydration makes it all but impossible to do complex testing without venturing into the browser-side</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: wondering if A/B vendors have data on how often experiments are launched at 100% as a fix, feature, etc.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Aly Cabral</b>: what are examples of client data that could not be passed to the server or edge?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: Aly: For example, if you want to have a conditional experiment based on your analytics (which is often outsourced and not accessible on the server), your ad provider, etc.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: what Ilya said, plus "data layers" to use tag manager parlance, targeting info that's brought in via JS and made available in some JS object/API</span></li><li class="c3 li-bullet-0"><span class="c1">... it's actually worsening with the restrictions from ITP and similar browser mitigations. more stuff is injected in the page now, harming cacheability</span></li><li class="c3 c25 li-bullet-0"><span class="c1"></span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: Agree strongly with the SPA comments. Gets extra complex when folks are using SSR + SPA. So you need to change both the SSR and the SPA navigation content.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: Product idea for CDNs. ;)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: CDNs can help but don't solve this problem</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Gotta be free (or cheap enough) too.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: Server side testing: possible and great, for those that have access to all necessary signals on the server *and* engineering resources to setup and run this.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: The CDN could segment the population and serve different A and B versions. &nbsp;They're independent of the platform used by the origin.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: That supposes that there is simply an "A" and "B" version - people who pay for e.g. Optimizely are not testing this way</span></li><li class="c3 li-bullet-0"><span class="c1">… imagine there are 3 experiments on a page with 2 variations each. it's a combinatorial explosion of permutations of every single page. if you fragment your CDN cache enough you might as well just turn off caching</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: As in A1+B1+...+Z2 vs. A2+B2+...Z1 vs ... , right?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: If a given page has 3 experiments, but the entire site has 2 "global" experiments (let's say a change to the masthead, and whether to pop some kind of promotion in a modal) - now you've got 5 on that page. and it's not uncommon for them all to have 2-4 variations.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Scott Connelly</b>: you could intercept the request and the server response to modify changes via reverse proxy</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: (Also, to be clear to those who don't know me well: I'm not trying to shoot down any ideas! These are all great considerations. just adding proprietary context I might have from working with big users A/B tests, and walls I ran into.)</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Scott Connelly</b>: No optimizely moves the script to the CDN</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Scott yes that's how Optimizely's web product works, I meant what we prototyped internally that was never productized :)</span></li></ul><h2 class="c10" id="h.d0oaj7ed2p0v"><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1QwEP4m-EneSYkW1QtD8SN7eAyjxsaobo6XIXMU3tDcU/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543363000&amp;usg=AOvVaw0LA9fxqT6P-BewTvFl-lyS">How client-side A/B testing works</a></span><span class="c21">&nbsp;- Optimizely - Greeshma Yellareddy</span></h2><ul class="c5 lst-kix_17xu3lqzjo8t-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Here to present how Optimizely implements A/B testing</span></li><li class="c2 li-bullet-0"><span class="c1">… In addition to previous benefits, short lived experiments can be done much faster without engineering input</span></li><li class="c2 li-bullet-0"><span class="c1">… Personalization - Using third-party integrations, if someone is from Microsoft, show this. &nbsp;These examples make it harder when it comes to server-side experiments.</span></li><li class="c2 li-bullet-0"><span class="c1">… Constraints: Has to preserve user experience, flashing can undermine integrity of the experimentation results. We cannot split the JS without risking it loading after the page is rendered. </span></li><li class="c2 li-bullet-0"><span class="c1">… Another constraint is customers don’t want to deal with nitty gritty details. &nbsp;Zero knowledge delivery.</span></li><li class="c2 li-bullet-0"><span class="c1">… Targeting decisions based on client-side attributes (cookies, IP, geo, device, browser, query string, URL, DOM, JS condition, Element present, localStorage, etc)</span></li><li class="c2 li-bullet-0"><span style="overflow: hidden;<b> display</b>: inline-block;<b> margin</b>: 0.00px 0.00px;<b> border</b>: 0.00px solid #000000;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);<b> width</b>: 624.00px;<b> height</b>: 352.00px;"><img alt="" src="./WebPerfWG open meeting - A_B testing_files/Screen Shot 2021-02-08 at 15.53.43.png" style="width: 624.00px;<b> height</b>: 352.00px; margin-left: 0.00px; margin-top: 0.00px;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></li></ul><ul class="c5 lst-kix_17xu3lqzjo8t-1 start"><li class="c3 li-bullet-0"><span class="c1">… Experiments that execute on certain DOM changes or JS conditions</span></li><li class="c3 li-bullet-0"><span class="c1">… Depend on localStorage for behavior changes (e.g. on 3rd visit)</span></li></ul><ul class="c5 lst-kix_17xu3lqzjo8t-0"><li class="c2 li-bullet-0"><span class="c1">… How does it work? &nbsp;One JavaScript snippet in the &lt;head&gt; of the website. Marketing folks can ask engineer to implement once, then perform all experiment changes throught the WYSIWYG editor</span></li><li class="c2 li-bullet-0"><span class="c1">… JS includes all code and data required to run tests (not just framework but the data as well), so all the experiments and variations</span></li><li class="c2 li-bullet-0"><span class="c1">… Performance challenges: Code necessary to evaluate is not cheap.</span></li><li class="c2 li-bullet-0"><span class="c1">… Can be any number of experiments (5-50+).</span></li><li class="c2 li-bullet-0"><span class="c1">… Changes themselves can be very large (non-engineers may not write optimized code)</span></li><li class="c2 li-bullet-0"><span class="c1">… Sync JavaScript is required to be effective</span></li><li class="c2 li-bullet-0"><span class="c1">… Experiment data changes frequently so JavaScript itself cannot be cached</span></li><li class="c2 li-bullet-0"><span class="c1">… CloudFlare Workers gives us a chance to improve our product</span></li><li class="c2 li-bullet-0"><span class="c1">… Edge node can execute faster, enables to send down just the minimal amount of Javascript</span></li><li class="c2 li-bullet-0"><span class="c1">… Many visitor attributes can still be done at the edge (IP, etc)</span></li><li class="c2 li-bullet-0"><span class="c1">… This is Optimizely Edge, first-party JS snippet loaded sync in the HEAD. &nbsp;Fetches from CloudFlare Worker CDN. Includes only code to execute in the current context of the visitor. That also means that there are a few things that cannot be done on the edge. Some of them can be done using a followup snippet.</span></li><li class="c2 li-bullet-0"><span class="c1">… Customers inject a small first-party script to avoid flicker </span></li><li class="c2 li-bullet-0"><span class="c1">… 230 KB of JavaScript is now close to 5-10 KB range, execution takes less than 5ms because it’s just applying</span></li><li class="c2 li-bullet-0"><span class="c1">… Inject a follow-on JavaScript snippet asynchronous, for tracking and other use cases that are not render blocking and can only be done on the client side (SPA related changes, JS conditions)</span></li><li class="c2 li-bullet-0"><span class="c1">… Removes some of the challenges, but there’s still a need to some sync JS loading, which changed frequently, so cannot be cached.</span></li><li class="c2 li-bullet-0"><span class="c1">… Still challenges around Security and evolving browser frameworks/platforms</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: One question from Pat is do we have a sense for how much pain is coming from A/B testing library plus data versus running client side code.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Patrick</b>: Try to figure out where the source of the pain is. &nbsp;Serving from 3P origin is some of the pain (SPOF, scheduling and stuff). &nbsp;If we can 1P serve and get it quickly in cache, how much of the A/B testing goes away, or is there still a lot of work going in the browser.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: I think we have to solve both problems. &nbsp;Serving from 1P reduces download time, but especially on mobile browsers JS execution time becomes a problem.</span></li><li class="c2 li-bullet-0"><span class="c1">… Some of our customers have about 100 experiments, which could be hundreds of milliseconds to evaluate</span></li><li class="c2 li-bullet-0"><span class="c1">… Edge takes all of that out (moves it to the edge)</span></li><li class="c2 li-bullet-0"><span class="c1">… Compute time on the edge is faster, and reduces bytes sent to client</span></li></ul><h3 class="c19" id="h.4z6om7hd1puk"><span class="c13">Chat conversation</span></h3><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span><b>Patrick Meenan</b>: </span><span class="c1">Do we have a sense for how much of the pain is in delivering the A/B library+configs vs running the client-side code? &nbsp;If it was all still client-side but same-origin we could keep all of the tooling and logic but would the gains be "enough"?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Optimizely prototyped this pretty substantially, the combinatorial explosion still harmed the CDN cache hit rate sufficiently to make it not worthwhile for the customers operating at the scale that need this</span></li><li class="c3 li-bullet-0"><span class="c1">Kenji, Ilya, Tim: +1 </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Optimizely has substantial data on delivery vs. execution, perhaps we could ask Greeshma to share some with the group after</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: I'd love to hear more on Tim's earlier comment of poorly implemented + large JS bundles that execute the transforms. Are there any low-hanging fruits there?</span></li><li class="c3 li-bullet-0"><span><b>Greeshma</b>: </span><span class="c20 c11">Optimizely's split between download + execution: Based on 12.5B data points, Download(p50): 52.8ms p75: 250ms JS Execution &nbsp;p50: 109ms p75: 226ms. This is for the non-Edge product</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: Super interesting, thanks Greeshma. Based on what you said earlier, it sounds like the edge product distribution should be significantly lower for the JS execution? Lower JS bundle, etc? Are we looking at order of magnitude diff?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: Would love to see any quantitative data on where we're seeing the highest perf "cost" (cold fetch + exec A/B testing library? active transformations? &nbsp;hard to generalize?). Anti-flicker timeouts could obviously be shorter. Edge-approach seems compelling for shrinking some of the JS bundle size at least.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kristofer Baxter</b>: @Greeshma when you say "download" do you mean the entire network connection including DNS resolution? Or is this purely transfer for established connections?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Kristofer it includes the end-to-end resource timing phases</span></li><li class="c3 li-bullet-0"><span><b>Greeshma Yellareddy</b>: </span><span class="c20 c11">For edge, p50 for download is 160ms, p75: 237; p50 for execution is 3ms, p75: 9ms</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Tim Kadlec</b>: Whew that execution improvement! :) </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: @kristofer, by download I mean resource timing's duration metric, which includes everything including DNS, SSL cert etc..</span></li><li class="c3 li-bullet-0"><span>… I might have missed some questions, please @ me if you have any more questions/request for data about Optimizely's implementation.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: That list of constraints on client-side only information seems critical for thinking about this problem space. Thanks for that! </span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1">… Would be interesting to get data on how often those "client side only" constraints are actually used and cannot be replaced with something more efficient</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Andy Davies</b>: Watching the DOM while it's being built leads to a *lot* of Mutation Events being fired</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1">… Something that more specifically targets DOM nodes would help in my view - lolhtml's method of using CSS selectors to target DOM nodes might be more effective</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Tim Kadlec</b>: There's been a lot of variance there, as you probably would expect, from org to org in my experience</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: yes, you'd definitely have to segment it by customer. it's hard even then because the content of the payload is constantly changing.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Andy Davies</b>: Providers to adding User Timing marks for key milestones would be hugely helpful</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Optimizely adds a couple I believe. I suspect Greeshma would be amenable to adding your specific requests Andy :)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Wondering about what fraction of Optimizely customers &nbsp;are taking advantage of this (Performance Edge). Is it free, or does one have to pay for the edge worker CPU time?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Scott Connelly</b>: upgrade</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: It's a paid product, they don't charge for CPU time or anything. It's a rather inexpensive add-on to their existing product lineup.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: +1. Curious to hear more about the limitations of Edge-based A/B testing. Anecdotally I've heard concern about CPU times leading to higher costs on the customer. Less insight into this for Optimizely.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: The bigger issue is that there is not feature parity, due to the limitations she mentioned about targeting etc</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Scott Connelly</b>: edge based compliant?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Scott like PCI or?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Scott Connelly</b>: yes-not sure</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: It could be. Optimizely's Edge-based product does not offer PCI compliance, but that was merely a product decision</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer Baxter</b>: As CWV evolves: Do we also need to solve the concern over the course of a visit versus an individual document visit?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: @Kristofer: yes. :) </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: @kristofer are you referring to how SPAs are treated by CWV? or for all sites in general (non SPAs as well) ?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kristofer Baxter</b>: Yes, all sites independent of technologies used by the domain.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Andy Davies</b>: Testing page-to-page flows on sites with anti-flicker snippets the blank screens are noticeable (often 1-2 secs) </span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Wondering if A/B solutions do anything fancy for subsequent navigations, I assume that the A/B selection would more or less &nbsp;result in the same outcomes but maybe not. Are there any optimizations done, if not should there be?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Kenji depends if you mean SPA or non-SPA</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: non-SPA</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Optimizely doesn't do any optimizations in that case, then. It doesn't know if the payload (experiments, targeting) changed between page navigations, so things need to be re-evaluated. It just relies on the fact that the bucketing decisions will be the same (consistent hashing on an ID in a first-party cookie). Optimizations could be done here, yeah.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andy Davies</b>: VWO generates a unique script for each page in a non-SPA case</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andrew Galloni</b>: Subsequent navigations or any event data is more part of the customer experience suite of products that decide the best next interaction or image to show. </span></li></ul><p class="c8 c25"><span class="c1"></span></p><h2 class="c10" id="h.ukhxosf1lzyy"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1M414_RHF--1IVyYb6V_9wxCsa-yRO5_HGsIvkY__sas/edit?usp%3Dsharing&amp;sa=D&amp;source=editors&amp;ust=1613054543377000&amp;usg=AOvVaw2OWyXIeaiAd9rQKDTlilI1">Google Optimize</a></span><span>&nbsp;- </span><span>Dimitris Dimitropoulos</span><span>&nbsp;- </span><span>Google Optimize</span></h2><ul class="c5 lst-kix_q3zi7brx38zu-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: Google Optimize launched in July 2016</span></li><li class="c2 li-bullet-0"><span class="c1">… Google Optimize also provides server-side experiments, but we’ll cover just Client-side today</span></li><li class="c2 li-bullet-0"><span class="c1">… Marketing and UX improvements efforts do matter a lot to performance of the site</span></li><li class="c2 li-bullet-0"><span class="c1">… Finding solutions to improve sites is better for the site and for users</span></li><li class="c2 li-bullet-0"><span class="c1">… Usage - 35% of top sites</span></li><li class="c2 li-bullet-0"><span class="c1">… How things works - marketer uses the editor to produce changes in the page, served in script to the browser to implement personalization.</span></li><li class="c2 li-bullet-0"><span class="c1">… Editor needs some knowledge, but there’s always a range of skills out there, and this tool can help bypass the bureaucracy</span></li><li class="c2 li-bullet-0"><span class="c1">… Targeting: URL and Audience based</span></li><li class="c2 li-bullet-0"><span class="c1">… Due to referrer policies the page’s URL can’t be reliably detected server-side (for a 3P script)</span></li><li class="c2 li-bullet-0"><span class="c1">… URL fragment also used for SPA sites</span></li><li class="c2 li-bullet-0"><span class="c1">… Abilities of a marketer to ask developer to only experiment in a particular city, they value the flexibility of the tool</span></li><li class="c2 li-bullet-0"><span class="c1">… Why do we need to go to the client for the targeting? &nbsp;Some signals are only in the client, integration with other 3P systems. &nbsp;Also is the user logged in?</span></li><li class="c2 li-bullet-0"><span class="c1">… Users install snippet in whole site</span></li><li class="c2 li-bullet-0"><span class="c1">… Many users prefer flexibility of installing via Google Tag Manager. &nbsp;Suboptimal situation from a performance point of view.</span></li><li class="c2 li-bullet-0"><span class="c1">… Make experiments in the Optimize UI and editor</span></li><li class="c2 li-bullet-0"><span class="c1">… Guide users to help limit harm to sites.</span></li><li class="c2 li-bullet-0"><span class="c1">… The script started as async, a module of Google Analytics. &nbsp;We now offer both async and sync. &nbsp;No dependency to Analytics.</span></li><li class="c2 li-bullet-0"><span class="c1">… Flicker: Use this term to refer to “flash of wrong content” -- Re-rendering is often unavoidable.</span></li><li class="c2 li-bullet-0"><span class="c1">… Where user sees wrong content and it changes</span></li><li class="c2 li-bullet-0"><span class="c1">… Once Optimize container loads, it adds CSS to hide (via opacity) all future unloaded content that will have to be changed. Also has the ability to hide only parts of the page.</span></li><li class="c2 li-bullet-0"><span class="c1">… But if the page renders first, there’s nothing that can be done</span></li><li class="c2 li-bullet-0"><span class="c1">… For SPAs you would hope developers would have more agency to perform page changes, but there’s still a need from marketers to perform page changes</span></li><li class="c2 li-bullet-0"><span class="c1">… How many experiments? Optimize free allows for up to 5 experiments running, that’s one of the reasons we don’t see a lot of parallel experiments. Users used to do few things with high impact.</span></li></ul><ul class="c5 lst-kix_q3zi7brx38zu-1 start"><li class="c3 li-bullet-0"><span class="c1">… Story: targeted a feature for COVID banners, with different banners based on geo. Difficult through the development process, but easy with personalization solutions.</span></li></ul><ul class="c5 lst-kix_q3zi7brx38zu-0"><li class="c2 li-bullet-0"><span class="c1">… See very often that businesses have inactive installs, where they don’t run experiments. Installing with Tag Manager gives them the flexibility to disable that.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: One question regarding usage, with GTM is due to convenience, but adds extra hop or two to actually load the render-blocking content. Are users using GTM aware of the cost that the convenience adds? &nbsp;Or are they not aware of implications?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: There’s a range, we try to warn in our documentation. &nbsp;Users may pay the cost of GTM if they’re just running the experiment for a week, then being able to remove it. &nbsp;So there’s a tradeoff there. We do try to warn them that they’ll get better performance using it directly.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: Sometimes people don’t run experiments for a long time, though they may have machinery in place. &nbsp;Can 3P respond with 204 No Content but a revalidate, where they could pick up new experiments when they run.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: Might make sense. &nbsp;Some details that have prevented us from doing that. &nbsp;Empty script is small, 20 KB or so. &nbsp;Might get more users wondering why it doesn’t work as they expect.</span></li><li class="c2 li-bullet-0"><span class="c1">… Also have an API client-side to know if experiments are running. &nbsp;Might be good for SPAs where experiments aren’t on the first rendering path.</span></li></ul><h3 class="c19" id="h.ezqbydd890ni"><span class="c13">Chat conversation</span></h3><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Interesting that Optimize puts an actual upper limit on file size. Optimizely probably should do that, haha.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Does optimizely also do the "single JS" approach (IIUC, this is packing all the experiments in one response)? Maybe that only works if the payload has a reasonable max size though.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: It does.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Paul Bernier</b>: @Kenji there are scenarios where the experience is dependent on mid-session actions (e.g. log-in, add to cart, ...). You often need additional logic and evaluation &nbsp;- past the first page load - to determine what the experience should be. This is not necessarily used to determine which variant to show, but to determine which Test user should fall in.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: The payload grows to whatever size the customer grows it to.. (empty is around 100kB gzipped)</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: why is empty so big?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Insufficiently granular engine for change application, event dispatching, etc.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: if there are expensive things (bytes, time) that the engine does because of gaps in the platform, we would love to learn more details.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Greeshma is the expert on that, would highly encourage you two to talk :)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>CP Clermont</b>: Common problem: Anti-flicker snippet is there even if no experiments are running.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Nathan Tate</b>: The anti-flicker snippet must be installed directly on the page, you don't want that loading asynchronously otherwise it defeats the purpose</span></li><li class="c3 li-bullet-0"><span class="c1">...Which naturally conflicts with the desire to know whether or not there's an experiment running</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: Indeed. I only want to highlight that we're ill equipped since usually the experiment definition is defined on the A/B testing platform, whereas the HTML is being served from another.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: you just need another anti-flicker snippet for that one... /s</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: There's no good way of conditionally having an anti-flicker snippet if and only if an experiment is running on that page. </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Nathan Tate</b>: Agreed. I'd be interested in a solution to that if possible, but some kind of network request would need to be made somewhere</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andy Davies</b>: I think there's a question of how often the anti-flicker snippet is actually needed even when there are experiments running</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: @Andy +1</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andy Davies</b>: @CP looks like Gymshark have removed their anti-flicker snippet</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: On the "no experiments" running for some time, I'd love to know if the runtime script could update its response with a noop response + S-W-R to revive it when an experiment becomes available.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Kenji it's pretty rare that someone has *no* experiments running, in my experience. having some with targeting conditions that ~never fire is more common, which can't be known until runtime, unfortunately</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: @Michael. But usually the snippet is installed on all pages. So whether you have experiments or not might be irrelevant. It's more about are you running experiments on _the current page?_ </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @CP Yes if it could be detected. That's how the Optimizely Edge product works. Serves like a couple hundred bytes or less if there's nothing on that page.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: @Michael that is pretty great. </span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: To anyone who has used such products: What exactly do the WYSIWGYG editors look like? &nbsp;Is it like devtools editor where you can manipulate dom/css? &nbsp;Is it specific to WYSIWYG editors for the original site impl/CMS?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Peter Perlepes</b>: @Michal They are more like a mix, you could rearrange stuff even by drag &amp; drop. But you can even modify the innerHTML of any element, add extra CSS &amp; JS on top. More oriented to a webmaster I would say</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: +1 what @Peter said - I'd add that some have functionality that allows developers to create reusable components for less technical operators of the software. "Widgets" to deploy things like modals, etc.</span></li></ul><h2 class="c10" id="h.wjbbutlslnvj"><span class="c4 c11"><a class="c6" href="https://www.google.com/url?q=https://docs.google.com/presentation/d/1rmVcHJtfJhFvASGfsbPjwguW7kP9D8Le0YW6zx_vk-0/edit%23slide%3Did.p&amp;sa=D&amp;source=editors&amp;ust=1613054543388000&amp;usg=AOvVaw1Ky6PZNw3XrpZkUCy8xkj_">Client-side A/B testing outside the client??</a></span><span class="c21">&nbsp;- Yoav Weiss</span></h2><ul class="c5 lst-kix_5z3ihgj0q4tq-0 start"><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: As we heard, client side A/B testing has tradeoffs. Can we as an industry do better, provide the benefits without the downsides?</span></li><li class="c2 li-bullet-0"><span class="c1">… Goals: Enable code-free experimentation</span></li><li class="c2 li-bullet-0"><span class="c1">… Enable the HTML to be transformed before reaching the browser</span></li><li class="c2 li-bullet-0"><span class="c1">… cacheing - When running multiple server-side experiments, we’re running risk of exploding the caching granularity and forcing caches to save a copy of transformed HTML per experiment</span></li><li class="c2 li-bullet-0"><span class="c1">… Either we abandon caching altogether, or we have caches for each experiments which results in reduced hit rates</span></li><li class="c2 li-bullet-0"><span class="c1">… Would like CDN flexibility, Greeshma talked about Edge side delivery, but if they depend on a single CDN that limits them from moving or using multi-CDN. &nbsp;This can reduce adoption.</span></li><li class="c2 li-bullet-0"><span class="c1">… Would like a solution that works for all CDNs or no CDN at all</span></li><li class="c2 li-bullet-0"><span class="c1">… Performance-oriented experimentation. &nbsp;Performance impact can throw off whatever results you get. Andy Davies wrote a blog post on such experimentation, but frameworks don’t give you that ability.</span></li><li class="c2 li-bullet-0"><span class="c1">… Two places we could perform transformations before they get to the user: at Edge, or before the HTML leaves the origin’s network, but after it leaves its caches</span></li><li class="c2 li-bullet-0"><span style="overflow: hidden;<b> display</b>: inline-block;<b> margin</b>: 0.00px 0.00px;<b> border</b>: 0.00px solid #000000;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);<b> width</b>: 624.00px;<b> height</b>: 350.67px;"><img alt="" src="./WebPerfWG open meeting - A_B testing_files/Screen Shot 2021-02-08 at 15.54.29.png" style="width: 624.00px;<b> height</b>: 350.67px; margin-left: 0.00px; margin-top: 0.00px;<b> transform</b>: rotate(0.00rad) translateZ(0px); -webkit-transform: rotate(0.00rad) translateZ(0px);" title=""></span></li><li class="c2 li-bullet-0"><span class="c1">… Option 1: Experiment proxy on origin network. &nbsp;Sub-options for each A/B framework implements, or a common code-base with plugins, or a common transformation language that a single proxy could apply at the origin</span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-1 start"><li class="c3 li-bullet-0"><span class="c1">… Pros: Low-level of industry consensus, doesn’t require a CDN.</span></li><li class="c3 li-bullet-0"><span class="c1">… Cons: Past experience (Mod_PageSpeed) is not encouraging, per-site install/maintenance costs</span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-0"><li class="c2 li-bullet-0"><span class="c1">… Option 2: CDN transformation service. &nbsp;Either provide off-the-shelf per-framework implementation for edge-side transformation, or for a common transformation language</span></li><li class="c2 li-bullet-0"><span class="c1">… Pros: CDNs can facilitate installation, cost is shared. &nbsp;</span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Cons</b>: Requires industry consensus (transformation language), per-site maint const (transferred to customers), per-site activation (slow adoption curve)</span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-0"><li class="c2 li-bullet-0"><span class="c1">… Alternatives: Can the browser help current frameworks? </span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-1 start"><li class="c3 li-bullet-0"><span class="c1">Render-blocking but not parser-blocking script loading?</span></li><li class="c3 li-bullet-0"><span class="c1">Primitives for client-side HTML transformation (that worked on first load).</span></li><li class="c3 li-bullet-0"><span class="c1">Could use network-wide solution as a serviceworker, but not viable because first-load doesn’t have SW active</span></li></ul><ul class="c5 lst-kix_5z3ihgj0q4tq-0"><li class="c2 li-bullet-0"><span class="c1">… Options are not necessarily mutually exclusive</span></li><li class="c2 li-bullet-0"><span class="c1">… Some transformations will always be on client side because there’s some info the client has (and we don’t want to send to the server)</span></li><li class="c2 li-bullet-0"><span class="c1">… Work towards both of those solutions</span></li><li class="c2 li-bullet-0"><span class="c1">… Brainstorm!</span></li><li class="c2 li-bullet-0"><span class="c1">… Questions to A/B providers: If there is a single transformation language that A/B testing frameworks emit, that can be run either client-side or edge/origin network, is that something that would be feasible/appealing or could this work?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: We had a hack project at Optimizely, for CMS. &nbsp;Created a language with CMSs could hook into and create their own implementation. &nbsp;But you still had WYSIWYG to make changes. It’s a possibility, but getting adoption would be biggest problem. Most customers don’t have the resources to create specific server side implementations.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: A standard language: Most tools have primitives that are equivalent I would guess, if that language would be something that the browser implemnts, that would always work, and then in some cases could also be done server side. &nbsp;There are other cases where you don’t really have the content on the server, but maybe it’s something that can work.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Ishan</b>: For a common transformation language, you want it to work flexibly across client and server and across SPA lifecycle. We have a lot of clients that need A/B testing in the context of SPAs. Our clients are IT dev teams, so it may be easier for them, but we still want a WYSIWYG solution. Intuition is there’s not one solution that solves all. &nbsp;Might be useful to prioritize.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Paul</b>: RE “Is this crazy” I don’t think it is. I think it goes back to the idea of compromise, between ease-of-use, comes with higher performance cost, or you go with performance and you’re typically on the developer side of things. &nbsp;I think it’s hard to rule out client-side techniques, to manipulate or create the experience. You can manipulate the HTML before it gets to the browser - in our case it’s regex on the HTML source before it reaches the browser. &nbsp;One thing is to use JS to modify an existing page, and it’s something else to swap the JS (HTML?) before it gets to the browser. A lot of gain in terms of performance, while maintaining ease of use. There’s a lot there. </span></li><li class="c2 li-bullet-0"><span class="c1">… The friction we see is you’re now talking to IT departments to get it in place, where the end-user (consumer) is more on the marketing side.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: You’re applying Regex on JavaScript is that on edge/origin/browser with hacks?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Paul</b>: Could be either-or -- anything in the flow of traffic (HTML, XML, JSON). &nbsp;Either on-prem (right after origin cache). &nbsp;Or it’s a private cloud or public cloud and you interface directly with CDN, but you’re modifying the source of the page (rather than modify JS).</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Michael</b>: Constraints with having a transformation language, I agree and also wonder if there are a few primitives supported but have an escape hatch for running custom JavaScript. &nbsp;Multiple sides of support needed here. &nbsp;CMS transforms. &nbsp;You also need customers that just need to paste in CSS or JavaScript, and not providing that would result in adoption issues. Providing an escape hatch that enables you to run arbitrary CSS or JS transforms would be good. Could be used to mitigate shortcomings, but point to happy path towards more performant things. &nbsp;Also helps gather use-cases for what transforms to support.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: In terms of what can browsers do to help. &nbsp;Something else browsers can do to load faster, consume less CPU, minimize towards zero the cost.</span></li><li class="c2 li-bullet-0"><span class="c1">… How do current implementations work? Are you registering MutationObservers and when DOM node is added you’re modifying them?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Yes, MO is registered and you mark it as already changed so you don’t re-apply. &nbsp;Before that, it was polling for an element to be present.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Michael</b>: Are there additional lifecycle hooks that the browser could provide that could allow for mechanical simplicity where A/B test could apply things at times where it’s more agreeable to the page?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: A few hooks, tracking itself when after everything is done. &nbsp;Most cases tracking is async but it can happen at a very critical event.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: On that front, if we were able to defer scripts and use perf APIs with buffered flags, is that sufficient?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Deferred is too early for tracking sometimes even. It’s not that critical. &nbsp;Use Perf APIS to find LongTasks. It doesn’t seem like a clean way. There should be a cleaner way to load those scripts.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Michael</b>: Question for browser folks: We talked about content-addressable JavaScript, inline snippets, there are discussions are around proxies and MITM rewriting. There’s a compromise in the case where a proxy injects the content especially in the SPA case where you can’t do the transformations yet. Problem with that is there’s some tipping point where putting it inline is worse that a script src tag from a CDN. &nbsp;To me a cacheable script could help parsing and execution as well as the bytes itself. Could the browser take into account hints that JS content provided on that front?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Currently a proposal for Web Shared libraries that can help with content-addressable content delivery. &nbsp;Main problem we haven’t solved yet are the privacy implications of having that resource in cache. Cached libraries can reveal information about the user, or the collection of them can be used as fingerprinting surface. </span></li><li class="c2 li-bullet-0"><span class="c1">… We've been thinking about the list of things that can be shared and prefetching those, so all users will have them in their caches in some sense. &nbsp;Not a trivial problem, but would help reduce download sizes here.</span></li></ul><a id="id.v0mx45y7rdsj"></a><ul class="c5 lst-kix_5z3ihgj0q4tq-0"><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: Idea of some sort of transformation language. If browsers start implementing that - CMS should have been playing bigger role in A/B experimentation story. &nbsp;There is no standard language for A/B tools to provide, so having one could help.</span></li><li class="c2 li-bullet-0"><span class="c1">… Persistence of experiment variance for users, most are using cookies. &nbsp;Raises questions about consent. Maybe it makes sense for browsers to help in that area? &nbsp;So we don’t have to touch cookie or localStorage for purposes of experimentation.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: If you wanted an arbitrary number of bits for variant ID, it could look an awful lot like a cookie.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: We keep talking about a split between different responsibilities. Marketing vs engineering. Is there a browser based solution that could take advantage of the different groups. Side-channel attached to HTML that applies as the document is streaming. &nbsp;This would allow those groups to work independently and would not require client-side JS to execute.</span></li><li class="c2 li-bullet-0"><span class="c1">… Also, we need to decide what are we trying to solve. Keep mixing concerns: solving for SPA, MPA, JAMstack. Can we limit the solution space to solve for a subset of those. &nbsp;Maybe won’t apply to SPA, which tends to need more framework &nbsp;knowledge, pretty complex. MPA solutions may be more simple, impact wider audiences faster.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Regarding first thought, sounds like common transformation language implemented in the browser. Is that what you had in mind?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kris</b>: The idea is to reduce the JS impact, and enable the browser to implement mutations in C++</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Browser would block rendering on this transformation language, but execution would be faster than arbitrary JavaScript, though JS could be added for things not accomplished by this. Interesting!</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: We’re using already established web tech, MutationObserver output.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: For things only working on server-side, there’s a big variety in the dynamic portions of sites, where SPAs are an extreme example. I did a lookup of a big sample of sites, large percentage of them are changing DOM in one way or another (social button, re-find your cart). This is a problem from A/B tools perspective, as you can’t tell users that they can change some parts because they are server-side generated but not other parts that are client-side generated, and often the tools don’t even know which is which.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: Idea is transformation language is implemented in the browser and applied on mutations as well, so can apply to dynamic parts. &nbsp;Probably problems with that but it’ll be an interesting space to pursue, because it will allow the browser to do work but be independent of backend technical implementation.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dimitris</b>: Sounds great!</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dave</b>: How would this proposed transformation language differ from CSS, because we’re already doing things like media-queries and responsive design.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: One example is how could CSS change the attribute of a custom element? An example I see frequently</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dave</b>: We could have elements hidden or shown based on cohort. &nbsp;Same way CSS does media queries but just based on a cookie.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: I think DOM transformation are key to what people are trying to do and it’s not a capability we want to give CSS</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Paul</b>: CSS I think would cover some of the use cases around look and feel, styling, but when you get into functionality the solutions go into DOM manipulation</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Ryosuke</b>: Not possible to add a new button somewhere with CSS. XSLT maybe it could do that.</span></li><li class="c2 li-bullet-0"><span class="c1">… Another thing missing is an async script that delays initial painting. The fact that scripts are doing it manually : avoid blocking parsing yet apply CSS to avoid painting. Seems like an API we could add and the browser could decide the script is taking too long, let’s not wait for them. One example shown was a page broken because the script took too long to load.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Agree. The only expectation is that I don’t think async is correct primitive, which is race-y. &nbsp;Fact that people resort to blocking scripts is a failure that doesn’t seem too hard to correct.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Ryosuke</b>: We need something that’s more async than sync but delays painting. &nbsp;What the exact timing is.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Conclusion: Revive XSLT &lt;/sarcasm&gt;</span></li><li class="c2 li-bullet-0"><span class="c1">… Would like to develop transformation language further</span></li><li class="c2 li-bullet-0"><span class="c1">… Kris, you mentioned MutationObserver output. Is that a thing?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: It’s the way the tool that we use work. We gather mutations and then apply the results on the client. May be overly chatty to adopt as is.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Would be interesting to study that. Is that part of the AMP project?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: The AMP project can use this from AMP caches as part of its experimentation framework</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Is it open source?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kris</b>: Not sure</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: Idea that you have a blocking script that is considered optional, useful to have a way for developer to say it should not take more than X milliseconds where then the browser would give up.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Thoughts on script loading that are not super baked. &nbsp;Today, developers need to say how script is meant to be loading where blocking/async are suboptimal and defer gives you a single point in time.</span></li><li class="c2 li-bullet-0"><span class="c1">… IMO, what we need are more options for defer semantics where browser can still render but not block parsing, or execute after first paint, or execute after onload on idle time. &nbsp;Move from “what” semantics to “when” and give people more control over the “when” and browsers can have more flexibility to when/how they’re loading for them to be ready on time.</span></li><li class="c2 li-bullet-0"><span class="c1">… Would allow for A/B testing to be render-blocking async with timeout</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: One other part not covered is what signal of variants to put a person in when they’re on the page. Sounds like this is something done in-script. If we’re delivering experiments through a “mutation” payload, it would need to support bucketing users into different experiments.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Talking about client-side variants?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer</b>: Mostly client-side. Currently there a script doeing mutation work. if the browser takes over that work, it needs to make decisions on which experiment the user belongs to. How would that be articulate?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: I’m guessing a lot of that is cookie-based, as well as additional checks done on the client.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Targeting is done cookie based, but decisions are done via JavaScript. &nbsp;Transformation would purely be client side. &nbsp;Would require some JavaScript for tracking purposes.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Goal is not necessarily to eliminate JavaScript altogether, but to remove HTML transformation and move that to a DSL. But if we need client-side code to decide which of the transformations should be applied. &nbsp;Potentially we could define buckets in transformation language, &nbsp;and then client-side JavaScript just says which bucket is picked.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: I would say that is not significantly less code, it’s the evaluation logic that takes the most time. &nbsp;So application of changes while slow, is not the biggest factor. Quoted numbers in chat RE application of changes in the Edge case, it takes only 5 ms, compared to 250ms.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Can you clarify evaluation logic?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Evaluation logic figures out which URL the visitor comes from, how many experiments are defined for this, if valid, which experiment does the user fall into. &nbsp;Then once you define that, you apply the logic.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: That’s the part that consumes the most on the client, and won’t change in the “transformation language” case.</span></li><li class="c2 li-bullet-0"><span class="c1">… Is there a way to delegate that logic to serving infrastructure. Assuming you had a list of client-side-only criteria. &nbsp;If someone doesn’t use them, server can decide on variant at serving time instead of the client</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: PerformanceEdge kinda does that. Lacks certain attributes like localStorage (behavioral).</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Do you know/can share how much pushback can you have from customers for not supporting client-only logic in those scenarios</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma</b>: Hadn’t supported SPA features (URL changed and DOM conditions)and that had a lot of pushback, since the major pushback is security (and we don’t support PCI on edge). </span></li><li class="c2 li-bullet-0"><span class="c1">… Other features - not a whole lot, any customer who cares about performance is willing to make that change. &nbsp;On landing page they implement Edge, on other pages they use traditional Optimizely. &nbsp;Most customers have been receptive. &nbsp;We do charge for it, so not all customers are moving to it.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Customers biased towards that solution are willing to tradeoff losing that customization for the performance</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Andy</b>: Going back to Melissa’s presentation and the business people creating these experiments, but we need to persuade testing providers to expose data about the impact of those experiments at runtime. So they know what impact their actions are having on their visitor. &nbsp;Give data to provide more informed decisions.</span></li><li class="c2 li-bullet-0"><span class="c1">… Interesting to understand how people doing edge-based testing (with Fast, Cloudflare, Akamai) are getting along and what restrictions they’re finding from that approach.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: Regarding the first point, it may be hard to get data without turning off A/B test completely</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Andy</b>: I’ve helped customers instrument this</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: Could A/B providers have standardized UserTiming?</span></li><li class="c2 li-bullet-0"><span class="c1">… Better if we had a common language</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Nic</b>: Maybe we can address some of the limitations of FCP/LCP to make sure the UX is reflected in the RUM data?</span></li></ul><a id="id.p3rnpc94zn3r"></a><ul class="c5 lst-kix_5z3ihgj0q4tq-0"><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: So if the first paint is followed by a blank page paint, maybe we should ignore the first one? Interesting!</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Michael</b>: Thanks to Yoav for getting us together today</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Ishan</b>: In the realm of A/B testing and you want to preserve caching, no standard way for edge to cache for specific sub-cookie in cookie string. So having a separate header for experiments would help. May make that type of splitting a lot more plausible to get better cache hit rates.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Yoav</b>: There was a Key proposal from mnot that might have helped over cookies, but that’s no longer a thing</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji</b>: Idea to have browser avoid painting of specific elements that would be affected by experimentation. Another thing we can explore.</span></li></ul><h3 class="c19" id="h.11v3qu7y6io8"><span class="c13">Chat conversation</span></h3><h4 class="c12" id="h.ct3z4ym1ylu"><span class="c15">Avoid blocking when no experiments are running</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: One question that came up earlier: how to avoid loading the blocking snippets if/when there are no experiments running? Any creative solutions?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: First one to adopt it get the perf wins.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: @ilya That's something CDN could help with. If there was a CDN pragma that could say easily, don't load this, would that suffice as a solution</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: @Ilya Inline script that picks a random number to segment population out of experiments altogether? &nbsp;Otherwise, are we just asking for server segmentation?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Colin Bendell</b>: is there a possibility of a Server-Hint that be standardized so that a platform like Shopify could utilize to noop the ab script injection?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: @Greeshma: not sure I follow, that seems reversed? The question is not "don't load please" but "if there are no experiments, please don't inject yourself". </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: @ilya I think the problem is that the script/snippet is included by the customer, either the CDN should remove it based on a "no experiment running, remove me" signal obtained async or, as I suggested, the script itself could respond with a "204 and a stale-while-revalidate: 1 day" so that it can auto-update itself when a new </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: Right, the mechanics of that is what I'm curious about.. CDN/edge node needs to know that there are no active experiments: what is that signaling mechanism?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: needs a way to know the backend endpoint, maybe it could just be fetching the script and seeing a 204 with some cache-control headers to know when to recheck.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: @Greeshma, @Dimitris: would love to hear your take on this. Is this plausible in practice?</span></li><li class="c3 li-bullet-0"><span class="c1">...From what I heard today, sounds like it *may* be possible in case of Optimizely Edge version, but afaik, it doesn't work with other implementations?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: Maybe a hint for the prioritization of some kind ;-)</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: "priority hints" but better (to be defiined).</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma</b>: @Ilya, even with Optimizely Edge, it's not possible from Optimizely's end. It will have to be the customer that implements something like it. What is easier for Optimizely or similar vendor, is to return an empty file.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: @Greeshma: interesting. If one wanted to implement the check at the edge, how would go about that today? I can hit Optimizely's... API to query status? Auth?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>CP Clermont</b>: RE: knowing if there's an experiment running on a page for anti-flicker snippets. I don't think there's a way out of having the origin being aware of the A/B testing platform. </span></li><li class="c3 li-bullet-0"><span class="c1">… Unless... it was possible for the anti-flicker snippet to know inline whether or not it should apply itself. Maybe with a resource hint? But then again, the origin would still need to be platform aware, because the origin would need to send the header. </span></li><li class="c3 li-bullet-0"><span class="c1">... + the CDN approach makes it so the A/B testing platform is part of the origin. Which might not work for reasons laid down earlier.</span></li><li class="c3 c25 li-bullet-0"><span class="c1"></span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: @Ilya: There is no one API for Optimizely currently. But Customer's experiment data is currently available on our CDN, because that's part of the code that is injected as part of the JS. So, auth should be no problem</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ilya Grigorik</b>: @Greeshma: I see, replaying just to make sure I grok it.. Given some customer ID in the current snippet, the edge can fetch experiment data from a well-known URL and check if it's empty. If so, ... X, otherwise, Y. </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: @Ilya: Yes. That's pretty much what edge does. Fetch the data from a well-defined URL</span></li></ul><h4 class="c12" id="h.3ncr1ietzh89"><span class="c15">Solution adoption</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Patrick Hamann</b>: I don't see how our past experience from Mod Pagespeed is comparable in this instance.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: I think Yoav was just comparing the experience of getting technology changes in folks origin stacks</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: @Patrick i took it to mean he's saying in terms of adoption</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Nathan Tate</b>: any solution that requires a client to "install" something correctly is a major pain-point.</span></li></ul><h4 class="c12" id="h.aiy53z4627fb"><span class="c15">Client Side HTML transformation</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: XSLT was a client side &amp; server side transformation language. it even had a point in the page lifecycle though Chrome removed it. </span></li><li class="c2 li-bullet-0"><span class="c1"><b>Paul Bernier</b>: SiteSpect essentially works like solution #1 and #2, leveraging a proxy to transform a page before it hits the browser. On-prem (on origin network), or Public/Private cloud, CDN integrated.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: I am curious about primitives for client-side HTML transformation. This could also help solve transformations in SPA frameworks, which is a big problem for A/B testing </span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span><b>Ishan Anand</b>: @Greeshma we've done a lot on that. XSLT, Tritium, Cheerio, etc. (eg. </span><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://www.smashingmagazine.com/2014/02/applying-xsl-transformations-to-responsive-web-design/&amp;sa=D&amp;source=editors&amp;ust=1613054543415000&amp;usg=AOvVaw1ELu7bE6lOzaoaHKof_-qR">https://www.smashingmagazine.com/2014/02/applying-xsl-transformations-to-responsive-web-design/</a></span><span class="c1">)</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: client side HTML transformation frameworks that is</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: I wonder if there's value in some sort of CSS "cohort" query, where the user is segmented via a header, cookie, or local storage, and the CSS controls element display based on it.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: I think CDNs would have an easier time filtering unused elements out of a page via CSS queries on the edge than via running Javascript on the edge.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: CSS does not really transform the DOM. XSLT does</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: Change custom elements to store attribute values as css variables? :P</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: sounds very hacky to me</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: CSS-data-binding. &nbsp;Very hacky.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: Isn't it a Pipeline language that's needed then?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer Baxter</b>: Q: Is there a smaller surface area (reduced type of transformation capabilities) that would cover a high percentage of the real world usecases but has much lower cost on the client?</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kristofer Baxter</b>: Q: On standardization, could there be a Web Standard that replays the output of a MutationObserver? (attached to the network request for the document to maintain separation of teams we heard about several times)</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: @kristofer that's very interesting. would like to know more if there are resources about that in AMP</span></li></ul><h4 class="c12" id="h.2xnomt1tuzf7"><span class="c15">Instrumenting cost of A/B testing</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Colin Bendell</b>: Another question related to the CWV discussion - we would love to have a consistent way to instrument the perf cost of the AB injected experiment. Ideally this would help us expose the cost of the experiments as implemented in RUM based tools</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Nathan Tate</b>: +1 Colin. A simple way of measuring the perf cost would be really useful. experiment is detected.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: Has this suggestion (exposing perf "cost" of tests in an understandable way during runtime) been considered by Optimize or Optimizely before?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: It's hard to do from within Optimizely. WE talk a lot about performance, we have our own RUM infrastructure. But to get the best impact, they'd have to do something like server side testing with and without Optimizely</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Colin Bendell</b>: consistent user-timing nomenclature? </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: Yeah, standardizing would help. We already have user timings in Optimizely. </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: Yep. Extremely manual. A common layer would be great.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Colin Bendell</b>: and a common definition of what is being measured </span></li></ul><h4 class="c12" id="h.s80oewi0ri4z"><span class="c15">Anti-flicker/better JS loading support</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Michael Hood</b>: render-blocking but not parser-blocking is very interesting.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Tim Kadlec</b>: ^^ That's kinda what Firefox does, right? Parses the HTML in the background, discards if the script messes with it (something like that at least....)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: I can't find any trace of it anymore but at one point we were considering providing proper anti-flicker support &nbsp;with a way to hold the first paint via JS (modulo reasonable timeout). Are there any needs on the anti-flicker front?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: Proper support for holding paint, especially if you could optionally scope it to a container element, would be pretty nice.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Element based would be nice. &nbsp;Especially, if the A/B solution had a way to quickly know that only Elements X,Y,Z might be impacted.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: I was actually thinking the opposite. &nbsp;an "optional" blocking script with a timeout would avoid the async/opacity dance but doesn't necessarily help perf</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: On the timeout idea for blocking but optional things, I have something to float. The other use case I have in mind is fonts (stylesheet =&gt; fonts chain) where you might be willing to wait for up to X but could do without.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: Load timeout as well as execution timeout attributes for scripts would be awesome.</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: execution timeout seems a bit harder to handle though: half done changes/... ?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: Execution timeout would be harder to do from browser perspective, since it can leave the script in a bad state</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: Execution timeout might be hard to pull off since they can't be rolled-back. &nbsp;Stopping execution arbitrarily in the middle of execution could be .... bad</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Bing Chen</b>: Timeout on the loading only. If execution is slow, it should be detected in preview.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: +1. Now that V8 tackles a lot more parse + compilation + processing work on background threads, setting a cut-off point without causing breakage would be pretty hard. Can't undo easily either.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: Load timeout wise, seems &nbsp;that there is interest. I will draft something and share back.</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: &lt;script optional timeout=N&gt; may be easier to tackle :) </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Andy Davies</b>: Other thing with a timeout is you'd want to know if the script timed out when analysing the tests results </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Greeshma Yellareddy</b>: Yes, having analytics about script time out would be crucial</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: makes sense, thanks!</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: @addy that would be the easiest, but more semantics on script attribute may be better</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: optional scripts probably need an inline fallback? &nbsp;In the case of a/b, presumably the fallback is to show what's there?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: Would onerror/onload be enough for the fallback of a timeout?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: I wouldn't say need as in required but that's a neat touch (inline and/or onerror).</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: @kenji Yeah I guess the "4s timeout" fallback today isn't implemented with onerror but just global state and conventions?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Addy Osmani</b>: I like the idea of treating timeouts as just an error (modulo unsure if A/B testing libraries + their customers would need additional signals it was specifically because of a timeout for RUM needs...)</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: a-b test itself is a "long-task"?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michael Hood</b>: @Addy there is already a 'survivor bias' of sorts for the providers' RUM data. a signal raised to customers on-page RUM would be useful. this already exists for XHR</span></li><li class="c3 li-bullet-0"><span class="c1">... onBigJSStateChange()? heh</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Kenji Baheux</b>: I would like to have an option that avoids retagging (updating the snippet). I was thinking that there could be an HTTP based &nbsp;option which vendors could use to specify what they believe is a reasonable timeout. This would be cached/reused perhaps with the cache-control semantics or something specific to that header. (HTTP purists might tell me that I'm crazy though)</span></li></ul><h4 class="c12" id="h.t2pm20pvehhq"><span class="c15">Edge-side experimentation</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Andy Davies</b>: @Patrick from Fastly or @Andrew from Cloudflare can you comment on how customers are experimenting at the edge</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Andrew Galloni</b>: @andy from my experience I've seen lots of performance related experiments to prototype perf improvements , similar to the ones you have tried. Such as moving 3rd party to 1st party, deferring scripts without having to change code on the origin. Trying different image formats etc. There is a lot of regional personalisation.</span></li></ul><h4 class="c12" id="h.lupoby9jwzvi"><span class="c15">Other</span></h4><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Michael Hood</b>: *Tons* of unanswered hypotheses around performance impact on UX, and performance impact of FE approaches.</span></li><li class="c2 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: AMP-style A/B testing?</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: @Carine you mean the AMP Cache splits the traffic? or how AMP does it now? (show/hide) or did you mean something else?</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: @Ishan I meant how AMP works now</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span><b>Michael Hood</b>: The consent issue is a great callout Dimitris - it causes a lot of inefficient tag manager based loading approaches. (response to </span><span class="c4"><a class="c6" href="https://docs.google.com/document/d/e/2PACX-1vScbVZC19jesKXrwmfOG3qSs7WF49wvqCRA0Vm3AElOofsN9g4t9v5AnQC_CX6aWgRzMdkAHhZI04IH/pub#id.v0mx45y7rdsj">this</a></span><span class="c1">)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Carine Bournez</b>: +1, we'd need to triage the use cases</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: I think I missed it, what is "PCI"? (was that the right acronym)</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span><b>Andy Davies</b>: </span><span class="c4"><a class="c6" href="https://www.google.com/url?q=https://www.pcisecuritystandards.org/&amp;sa=D&amp;source=editors&amp;ust=1613054543426000&amp;usg=AOvVaw34UGZzm7_9Ibf4kwEB0KyI">https://www.pcisecuritystandards.org/</a></span></li><li class="c3 li-bullet-0"><span class="c1"><b>Ishan Anand</b>: standard you need to follow to accept credit cards on a website</span></li><li class="c3 li-bullet-0"><span class="c1"><b>Melissa Mitchell</b>: PCI is a security standard for a lot of finance transactions. </span></li><li class="c3 li-bullet-0"><span class="c1"><b>Michal Mocny</b>: tyvm</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: I can't remember the name of the CSP-like performance header to block slow ads, etc. &nbsp;If it's still a thing, could it be extended to specify scripts for specific origins? &nbsp;Still requires adding a header but it's outside of the markup</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-1 start"><li class="c3 li-bullet-0"><span class="c1"><b>Patrick Meenan</b>: Ahh, Feature policy</span></li></ul><ul class="c5 lst-kix_cl521vqmc48c-0"><li class="c2 li-bullet-0"><span class="c1"><b>Dave Pifke</b>: Extend NEL for slow loading resources maybe?</span></li><li class="c2 li-bullet-0"><span><b>Nicolás Peña Moreno</b>: I believe I changed the impl already. Is there an easy way to test? (referring to </span><span class="c4"><a class="c6" href="https://docs.google.com/document/d/e/2PACX-1vScbVZC19jesKXrwmfOG3qSs7WF49wvqCRA0Vm3AElOofsN9g4t9v5AnQC_CX6aWgRzMdkAHhZI04IH/pub#id.p3rnpc94zn3r">this</a></span><span class="c1">)</span></li></ul><p class="c8 c25"><span class="c1"></span></p></div></div><div id="footer"><span>Published by <a target="_blank" title="Learn more about Google Drive" href="https://docs.google.com/">Google Drive</a></span><span class="dash">–</span><a href="https://docs.google.com/u/0/abuse?id=AKkXjozdVG9Dv5tPbZuaAPqA2FCpQhy87IqKap5bWiBhp_FVjLsML4R_fEIIhWtpkD90Sa9cfTiIYYAfrYL5FD0:0">Report Abuse</a></div><script type="text/javascript" nonce="">(function(){/*

 Copyright The Closure Library Authors.
 SPDX-License-Identifier: Apache-2.0
*/
var aa="function"==typeof Object.defineProperties?Object.defineProperty:function(a,b,c){if(a==Array.prototype||a==Object.prototype)return a;a[b]=c.value;return a};function ba(a){a=["object"==typeof globalThis&&globalThis,a,"object"==typeof window&&window,"object"==typeof self&&self,"object"==typeof global&&global];for(var b=0;b<a.length;++b){var c=a[b];if(c&&c.Math==Math)return c}throw Error("Cannot find global object");}var ca=ba(this);
function da(a,b){if(b)a:{var c=ca;a=a.split(".");for(var d=0;d<a.length-1;d++){var e=a[d];if(!(e in c))break a;c=c[e]}a=a[a.length-1];d=c[a];b=b(d);b!=d&&null!=b&&aa(c,a,{configurable:!0,writable:!0,value:b})}}var ea="function"==typeof Object.create?Object.create:function(a){function b(){}b.prototype=a;return new b},fa;
if("function"==typeof Object.setPrototypeOf)fa=Object.setPrototypeOf;else{var ha;a:{var ia={a:!0},ja={};try{ja.__proto__=ia;ha=ja.a;break a}catch(a){}ha=!1}fa=ha?function(a,b){a.__proto__=b;if(a.__proto__!==b)throw new TypeError(a+" is not extensible");return a}:null}var ka=fa;
function h(a,b){a.prototype=ea(b.prototype);a.prototype.constructor=a;if(ka)ka(a,b);else for(var c in b)if("prototype"!=c)if(Object.defineProperties){var d=Object.getOwnPropertyDescriptor(b,c);d&&Object.defineProperty(a,c,d)}else a[c]=b[c];a.o=b.prototype}da("Object.is",function(a){return a?a:function(b,c){return b===c?0!==b||1/b===1/c:b!==b&&c!==c}});var l=this||self;function la(){}function ma(a){var b=typeof a;return"object"==b&&null!=a||"function"==b}
function na(a,b){var c=Array.prototype.slice.call(arguments,1);return function(){var d=c.slice();d.push.apply(d,arguments);return a.apply(this,d)}}function oa(a,b){function c(){}c.prototype=b.prototype;a.o=b.prototype;a.prototype=new c;a.prototype.constructor=a;a.B=function(d,e,f){for(var g=Array(arguments.length-2),k=2;k<arguments.length;k++)g[k-2]=arguments[k];return b.prototype[e].apply(d,g)}}function pa(a){return a};var qa=Array.prototype.indexOf?function(a,b){return Array.prototype.indexOf.call(a,b,void 0)}:function(a,b){if("string"===typeof a)return"string"!==typeof b||1!=b.length?-1:a.indexOf(b,0);for(var c=0;c<a.length;c++)if(c in a&&a[c]===b)return c;return-1};function ra(a,b,c){for(var d in a)b.call(c,a[d],d,a)};var m;function p(a,b){this.i=a===sa&&b||"";this.j=ta}p.prototype.h=!0;p.prototype.g=function(){return this.i};var ta={},sa={};var ua=String.prototype.trim?function(a){return a.trim()}:function(a){return/^[\s\xa0]*([\s\S]*?)[\s\xa0]*$/.exec(a)[1]},va=/&/g,wa=/</g,xa=/>/g,ya=/"/g,za=/'/g,Aa=/\x00/g,Ba=/[\x00&<>"']/;function Ca(a,b){return a<b?-1:a>b?1:0};function q(a,b){this.i=b===r?a:""}q.prototype.h=!0;q.prototype.g=function(){return this.i.toString()};q.prototype.toString=function(){return this.i.toString()};function u(a){return a instanceof q&&a.constructor===q?a.i:"type_error:SafeUrl"}
var Da=/^(?:audio\/(?:3gpp2|3gpp|aac|L16|midi|mp3|mp4|mpeg|oga|ogg|opus|x-m4a|x-matroska|x-wav|wav|webm)|font\/\w+|image\/(?:bmp|gif|jpeg|jpg|png|tiff|webp|x-icon)|video\/(?:mpeg|mp4|ogg|webm|quicktime|x-matroska))(?:;\w+=(?:\w+|"[\w;,= ]+"))*$/i,Ea=/^data:(.*);base64,[a-z0-9+\/]+=*$/i,Fa=/^(?:(?:https?|mailto|ftp):|[^:/?#]*(?:[/?#]|$))/i;function Ga(a){if(a instanceof q)return a;a="object"==typeof a&&a.h?a.g():String(a);Fa.test(a)||(a="about:invalid#zClosurez");return new q(a,r)}
var r={},Ha=new q("about:invalid#zClosurez",r);var v;a:{var Ia=l.navigator;if(Ia){var Ja=Ia.userAgent;if(Ja){v=Ja;break a}}v=""}function w(a){return-1!=v.indexOf(a)};function x(a,b,c){this.i=c===Ka?a:""}x.prototype.h=!0;x.prototype.g=function(){return this.i.toString()};x.prototype.toString=function(){return this.i.toString()};var Ka={};function La(a,b,c,d){a=a instanceof q?a:Ga(a);b=b||l;c=c instanceof p?c instanceof p&&c.constructor===p&&c.j===ta?c.i:"type_error:Const":c||"";return void 0!==d?b.open(u(a),c,d,void 0):b.open(u(a),c)};function Ma(a){Ma[" "](a);return a}Ma[" "]=la;function y(a,b,c){return Object.prototype.hasOwnProperty.call(a,b)?a[b]:a[b]=c(b)};var Na=w("Opera"),z=w("Trident")||w("MSIE"),Oa=w("Edge"),Pa=Oa||z,Qa=w("Gecko")&&!(-1!=v.toLowerCase().indexOf("webkit")&&!w("Edge"))&&!(w("Trident")||w("MSIE"))&&!w("Edge"),Ra=-1!=v.toLowerCase().indexOf("webkit")&&!w("Edge"),Sa=w("Macintosh");function Ta(){var a=l.document;return a?a.documentMode:void 0}var Ua;
a:{var Va="",Wa=function(){var a=v;if(Qa)return/rv:([^\);]+)(\)|;)/.exec(a);if(Oa)return/Edge\/([\d\.]+)/.exec(a);if(z)return/\b(?:MSIE|rv)[: ]([^\);]+)(\)|;)/.exec(a);if(Ra)return/WebKit\/(\S+)/.exec(a);if(Na)return/(?:Version)[ \/]?(\S+)/.exec(a)}();Wa&&(Va=Wa?Wa[1]:"");if(z){var Xa=Ta();if(null!=Xa&&Xa>parseFloat(Va)){Ua=String(Xa);break a}}Ua=Va}var Ya=Ua,Za={},$a;if(l.document&&z){var ab=Ta();$a=ab?ab:parseInt(Ya,10)||void 0}else $a=void 0;var bb=$a;function cb(a){return y(a.prototype,"$$generatedClassName",function(){return"Class$obf_"+{valueOf:function(){return++db}}})}var db=1E3;function A(){}A.prototype.u=function(){return this.j||(Object.defineProperties(this,{j:{value:++eb,enumerable:!1}}),this.j)};A.prototype.toString=function(){var a=B(fb(gb(this.constructor)))+"@";var b=(this.u()>>>0).toString(16);return a+B(b)};function D(){}h(D,A);D.prototype.i=function(a){this.h=a;if(a instanceof Object)try{a.A=this}catch(b){}};function hb(a){a.h instanceof Error&&(Error.captureStackTrace?Error.captureStackTrace(a.h):a.h.stack=Error().stack)}D.prototype.toString=function(){var a=fb(gb(this.constructor)),b=this.l;return null==b?a:B(a)+": "+B(b)};function ib(){}h(ib,D);function jb(){}h(jb,ib);var eb=0;function E(){}h(E,jb);E.prototype.i=function(a){jb.prototype.i.call(this,Object.is(this.g,"__noinit__")?a:this.g)};function kb(){}h(kb,E);function lb(a,b){return"string"==typeof a?a.charCodeAt(b):a.g(b)};function B(a){return null==a?"null":a.toString()}function mb(a){return 65536<=a?B(String.fromCharCode((55296+((a-65536|0)>>10&1023)|0)&65535))+B(String.fromCharCode((56320+((a-65536|0)&1023)|0)&65535)):String.fromCharCode(a&65535)}function nb(a,b){var c=b,d=a.length,e;b=lb(a,(e=c,c=c+1|0,e));var f;if(e=55296<=b&&56319>=b&&c<d)a=f=lb(a,c),e=56320<=a&&57343>=a;var g;e?g=65536+((b&1023)<<10)+(f&1023)|0:g=b;return g};function ob(a,b){this.h=a;this.g=b}h(ob,A);function fb(a){var b=cb(a.h);0!=a.g&&(b="L"+B(b)+";");a=a.g;for(var c="",d=0;d<a;d=d+1|0)c=B(c)+"[";return B(c)+B(b)}ob.prototype.toString=function(){return"class "+B(fb(this))};function gb(a){var b=0;return y(a.prototype,"$$class/"+b,function(){return new ob(a,b)})};function pb(a,b){return null==a?a:b?decodeURI(a):decodeURIComponent(a)};var qb=/^(?:([^:/?#.]+):)?(?:\/\/(?:([^/?#]*)@)?([^/#?]*?)(?::([0-9]+))?(?=[/#?]|$))?([^?#]+)?(?:\?([^#]*))?(?:#([\S\s]*))?$/;function rb(a){a=qb.exec(a);for(var b=[],c=0;7>=c;c=c+1|0)a.length<=c||null==a[c]?b.push(null):b.push(a[c]);return b}
function sb(a,b){var c=a.indexOf(mb(35));c=0>c?a.length:c;a:{var d=0;for(var e=b.length;0<(d=a.indexOf(b,d))&&d<c;){var f=nb(a,d-1|0);if(38==f||63==f){if((d+e|0)>=a.length)break a;f=nb(a,d+e|0);if(61==f||38==f||35==f)break a}d=d+(e+1)|0}d=-1}if(0>d)return null;e=a.indexOf(mb(38),d);if(0>e||e>c)e=c;d=d+(b.length+1)|0;b=Math.min(a.length,d);a=a.substr(b,Math.min(a.length,Math.max(d,e))-b|0);c=" ";for(b=0;0<=(b=c.indexOf("\\",b));)36==c.charCodeAt(b+1|0)?(d=B(c.substr(0,b|0))+"$",e=b=b+1|0,c=d+B(c.substr(e))):
(d=B(c.substr(0,b|0)),e=b=b+1|0,c=d+B(c.substr(e)));a=a.replace(/\+/g,c);return pb(a,!1)};function F(a,b){this.h=b;for(var c=[],d=!0,e=a.length-1;0<=e;e--){var f=a[e]|0;d&&f==b||(c[e]=f,d=!1)}this.g=c}var tb={};function ub(a){return-128<=a&&128>a?y(tb,a,function(b){return new F([b|0],0>b?-1:0)}):new F([a|0],0>a?-1:0)}function G(a){if(isNaN(a)||!isFinite(a))return H;if(0>a)return I(G(-a));for(var b=[],c=1,d=0;a>=c;d++)b[d]=a/c|0,c*=4294967296;return new F(b,0)}var H=ub(0),J=ub(1),vb=ub(16777216);
function K(a){if(-1==a.h)return-K(I(a));for(var b=0,c=1,d=0;d<a.g.length;d++){var e=L(a,d);b+=(0<=e?e:4294967296+e)*c;c*=4294967296}return b}F.prototype.toString=function(a){a=a||10;if(2>a||36<a)throw Error("radix out of range: "+a);if(M(this))return"0";if(-1==this.h)return"-"+I(this).toString(a);for(var b=G(Math.pow(a,6)),c=this,d="";;){var e=wb(c,b).g;c=N(c,O(e,b));var f=((0<c.g.length?c.g[0]:c.h)>>>0).toString(a);c=e;if(M(c))return f+d;for(;6>f.length;)f="0"+f;d=f+d}};
function L(a,b){return 0>b?0:b<a.g.length?a.g[b]:a.h}function M(a){if(0!=a.h)return!1;for(var b=0;b<a.g.length;b++)if(0!=a.g[b])return!1;return!0}function P(a,b){a=N(a,b);return-1==a.h?-1:M(a)?0:1}function I(a){for(var b=a.g.length,c=[],d=0;d<b;d++)c[d]=~a.g[d];return(new F(c,~a.h)).add(J)}F.prototype.abs=function(){return-1==this.h?I(this):this};
F.prototype.add=function(a){for(var b=Math.max(this.g.length,a.g.length),c=[],d=0,e=0;e<=b;e++){var f=d+(L(this,e)&65535)+(L(a,e)&65535),g=(f>>>16)+(L(this,e)>>>16)+(L(a,e)>>>16);d=g>>>16;f&=65535;g&=65535;c[e]=g<<16|f}return new F(c,c[c.length-1]&-2147483648?-1:0)};function N(a,b){return a.add(I(b))}
function O(a,b){if(M(a)||M(b))return H;if(-1==a.h)return-1==b.h?O(I(a),I(b)):I(O(I(a),b));if(-1==b.h)return I(O(a,I(b)));if(0>P(a,vb)&&0>P(b,vb))return G(K(a)*K(b));for(var c=a.g.length+b.g.length,d=[],e=0;e<2*c;e++)d[e]=0;for(e=0;e<a.g.length;e++)for(var f=0;f<b.g.length;f++){var g=L(a,e)>>>16,k=L(a,e)&65535,t=L(b,f)>>>16,n=L(b,f)&65535;d[2*e+2*f]+=k*n;Q(d,2*e+2*f);d[2*e+2*f+1]+=g*n;Q(d,2*e+2*f+1);d[2*e+2*f+1]+=k*t;Q(d,2*e+2*f+1);d[2*e+2*f+2]+=g*t;Q(d,2*e+2*f+2)}for(e=0;e<c;e++)d[e]=d[2*e+1]<<16|
d[2*e];for(e=c;e<2*c;e++)d[e]=0;return new F(d,0)}function Q(a,b){for(;(a[b]&65535)!=a[b];)a[b+1]+=a[b]>>>16,a[b]&=65535,b++}function S(a,b){this.g=a;this.h=b}
function wb(a,b){if(M(b))throw Error("division by zero");if(M(a))return new S(H,H);if(-1==a.h)return b=wb(I(a),b),new S(I(b.g),I(b.h));if(-1==b.h)return b=wb(a,I(b)),new S(I(b.g),b.h);if(30<a.g.length){if(-1==a.h||-1==b.h)throw Error("slowDivide_ only works with positive integers.");for(var c=J,d=b;0>=P(d,a);)c=xb(c,1),d=xb(d,1);var e=T(c,1),f=T(d,1);d=T(d,2);for(c=T(c,2);!M(d);){var g=f.add(d);0>=P(g,a)&&(e=e.add(c),f=g);d=T(d,1);c=T(c,1)}b=N(a,O(e,b));return new S(e,b)}for(e=H;0<=P(a,b);){c=Math.max(1,
Math.floor(K(a)/K(b)));d=Math.ceil(Math.log(c)/Math.LN2);d=48>=d?1:Math.pow(2,d-48);f=G(c);for(g=O(f,b);-1==g.h||0<P(g,a);)c-=d,f=G(c),g=O(f,b);M(f)&&(f=J);e=e.add(f);a=N(a,g)}return new S(e,a)}F.prototype.and=function(a){for(var b=Math.max(this.g.length,a.g.length),c=[],d=0;d<b;d++)c[d]=L(this,d)&L(a,d);return new F(c,this.h&a.h)};F.prototype.or=function(a){for(var b=Math.max(this.g.length,a.g.length),c=[],d=0;d<b;d++)c[d]=L(this,d)|L(a,d);return new F(c,this.h|a.h)};
F.prototype.xor=function(a){for(var b=Math.max(this.g.length,a.g.length),c=[],d=0;d<b;d++)c[d]=L(this,d)^L(a,d);return new F(c,this.h^a.h)};function xb(a,b){var c=b>>5;b%=32;for(var d=a.g.length+c+(0<b?1:0),e=[],f=0;f<d;f++)e[f]=0<b?L(a,f-c)<<b|L(a,f-c-1)>>>32-b:L(a,f-c);return new F(e,a.h)}function T(a,b){var c=b>>5;b%=32;for(var d=a.g.length-c,e=[],f=0;f<d;f++)e[f]=0<b?L(a,f+c)>>>b|L(a,f+c+1)<<32-b:L(a,f+c);return new F(e,a.h)};N(xb(J,32),J);N(xb(J,128),J);function yb(a){a&&"function"==typeof a.v&&a.v()};function U(){this.h=this.h;this.g=this.g}U.prototype.h=!1;U.prototype.v=function(){this.h||(this.h=!0,this.j())};U.prototype.j=function(){if(this.g)for(;this.g.length;)this.g.shift()()};var zb=!z||9<=Number(bb),Ab=!z||9<=Number(bb),Bb=z&&!y(Za,"9",function(){for(var a=0,b=ua(String(Ya)).split("."),c=ua("9").split("."),d=Math.max(b.length,c.length),e=0;0==a&&e<d;e++){var f=b[e]||"",g=c[e]||"";do{f=/(\d*)(\D*)(.*)/.exec(f)||["","","",""];g=/(\d*)(\D*)(.*)/.exec(g)||["","","",""];if(0==f[0].length&&0==g[0].length)break;a=Ca(0==f[1].length?0:parseInt(f[1],10),0==g[1].length?0:parseInt(g[1],10))||Ca(0==f[2].length,0==g[2].length)||Ca(f[2],g[2]);f=f[3];g=g[3]}while(0==a)}return 0<=a}),
Cb=function(){if(!l.addEventListener||!Object.defineProperty)return!1;var a=!1,b=Object.defineProperty({},"passive",{get:function(){a=!0}});try{l.addEventListener("test",la,b),l.removeEventListener("test",la,b)}catch(c){}return a}();function Db(a,b){this.type=a;this.g=this.target=b;this.defaultPrevented=!1}Db.prototype.h=function(){this.defaultPrevented=!0};function V(a,b){Db.call(this,a?a.type:"");this.relatedTarget=this.g=this.target=null;this.button=this.screenY=this.screenX=this.clientY=this.clientX=0;this.key="";this.metaKey=this.shiftKey=this.altKey=this.ctrlKey=!1;this.state=null;this.pointerId=0;this.pointerType="";this.i=null;if(a){var c=this.type=a.type,d=a.changedTouches&&a.changedTouches.length?a.changedTouches[0]:null;this.target=a.target||a.srcElement;this.g=b;if(b=a.relatedTarget){if(Qa){a:{try{Ma(b.nodeName);var e=!0;break a}catch(f){}e=
!1}e||(b=null)}}else"mouseover"==c?b=a.fromElement:"mouseout"==c&&(b=a.toElement);this.relatedTarget=b;d?(this.clientX=void 0!==d.clientX?d.clientX:d.pageX,this.clientY=void 0!==d.clientY?d.clientY:d.pageY,this.screenX=d.screenX||0,this.screenY=d.screenY||0):(this.clientX=void 0!==a.clientX?a.clientX:a.pageX,this.clientY=void 0!==a.clientY?a.clientY:a.pageY,this.screenX=a.screenX||0,this.screenY=a.screenY||0);this.button=a.button;this.key=a.key||"";this.ctrlKey=a.ctrlKey;this.altKey=a.altKey;this.shiftKey=
a.shiftKey;this.metaKey=a.metaKey;this.pointerId=a.pointerId||0;this.pointerType="string"===typeof a.pointerType?a.pointerType:Eb[a.pointerType]||"";this.state=a.state;this.i=a;a.defaultPrevented&&V.o.h.call(this)}}oa(V,Db);var Fb=[1,4,2],Eb={2:"touch",3:"pen",4:"mouse"};V.prototype.h=function(){V.o.h.call(this);var a=this.i;if(a.preventDefault)a.preventDefault();else if(a.returnValue=!1,Bb)try{if(a.ctrlKey||112<=a.keyCode&&123>=a.keyCode)a.keyCode=-1}catch(b){}};var Gb="closure_listenable_"+(1E6*Math.random()|0);var Hb=0;function Ib(a,b,c,d,e){this.listener=a;this.g=null;this.src=b;this.type=c;this.capture=!!d;this.h=e;this.key=++Hb;this.m=this.s=!1}function Jb(a){a.m=!0;a.listener=null;a.g=null;a.src=null;a.h=null};function Kb(a){this.src=a;this.g={};this.h=0}Kb.prototype.add=function(a,b,c,d,e){var f=a.toString();a=this.g[f];a||(a=this.g[f]=[],this.h++);var g;a:{for(g=0;g<a.length;++g){var k=a[g];if(!k.m&&k.listener==b&&k.capture==!!d&&k.h==e)break a}g=-1}-1<g?(b=a[g],c||(b.s=!1)):(b=new Ib(b,this.src,f,!!d,e),b.s=c,a.push(b));return b};function Lb(a,b){var c=b.type;if(c in a.g){var d=a.g[c],e=qa(d,b),f;(f=0<=e)&&Array.prototype.splice.call(d,e,1);f&&(Jb(b),0==a.g[c].length&&(delete a.g[c],a.h--))}};var Mb="closure_lm_"+(1E6*Math.random()|0),Nb={},Ob=0;function Pb(a,b,c,d,e){if(d&&d.once)return Qb(a,b,c,d,e);if(Array.isArray(b)){for(var f=0;f<b.length;f++)Pb(a,b[f],c,d,e);return null}c=Rb(c);return a&&a[Gb]?a.g.add(String(b),c,!1,ma(d)?!!d.capture:!!d,e):Sb(a,b,c,!1,d,e)}
function Sb(a,b,c,d,e,f){if(!b)throw Error("Invalid event type");var g=ma(e)?!!e.capture:!!e,k=Tb(a);k||(a[Mb]=k=new Kb(a));c=k.add(b,c,d,g,f);if(c.g)return c;d=Ub();c.g=d;d.src=a;d.listener=c;if(a.addEventListener)Cb||(e=g),void 0===e&&(e=!1),a.addEventListener(b.toString(),d,e);else if(a.attachEvent)a.attachEvent(Vb(b.toString()),d);else if(a.addListener&&a.removeListener)a.addListener(d);else throw Error("addEventListener and attachEvent are unavailable.");Ob++;return c}
function Ub(){var a=Wb,b=Ab?function(c){return a.call(b.src,b.listener,c)}:function(c){c=a.call(b.src,b.listener,c);if(!c)return c};return b}function Qb(a,b,c,d,e){if(Array.isArray(b)){for(var f=0;f<b.length;f++)Qb(a,b[f],c,d,e);return null}c=Rb(c);return a&&a[Gb]?a.g.add(String(b),c,!0,ma(d)?!!d.capture:!!d,e):Sb(a,b,c,!0,d,e)}
function Xb(a){if("number"!==typeof a&&a&&!a.m){var b=a.src;if(b&&b[Gb])Lb(b.g,a);else{var c=a.type,d=a.g;b.removeEventListener?b.removeEventListener(c,d,a.capture):b.detachEvent?b.detachEvent(Vb(c),d):b.addListener&&b.removeListener&&b.removeListener(d);Ob--;(c=Tb(b))?(Lb(c,a),0==c.h&&(c.src=null,b[Mb]=null)):Jb(a)}}}function Vb(a){return a in Nb?Nb[a]:Nb[a]="on"+a}
function Yb(a,b,c,d){var e=!0;if(a=Tb(a))if(b=a.g[b.toString()])for(b=b.concat(),a=0;a<b.length;a++){var f=b[a];f&&f.capture==c&&!f.m&&(f=Zb(f,d),e=e&&!1!==f)}return e}function Zb(a,b){var c=a.listener,d=a.h||a.src;a.s&&Xb(a);return c.call(d,b)}
function Wb(a,b){if(a.m)return!0;if(!Ab){if(!b)a:{b=["window","event"];for(var c=l,d=0;d<b.length;d++)if(c=c[b[d]],null==c){b=null;break a}b=c}d=b;b=new V(d,this);c=!0;if(!(0>d.keyCode||void 0!=d.returnValue)){a:{var e=!1;if(0==d.keyCode)try{d.keyCode=-1;break a}catch(g){e=!0}if(e||void 0==d.returnValue)d.returnValue=!0}d=[];for(e=b.g;e;e=e.parentNode)d.push(e);a=a.type;for(e=d.length-1;0<=e;e--){b.g=d[e];var f=Yb(d[e],a,!0,b);c=c&&f}for(e=0;e<d.length;e++)b.g=d[e],f=Yb(d[e],a,!1,b),c=c&&f}return c}return Zb(a,
new V(b,this))}function Tb(a){a=a[Mb];return a instanceof Kb?a:null}var $b="__closure_events_fn_"+(1E9*Math.random()>>>0);function Rb(a){if("function"===typeof a)return a;a[$b]||(a[$b]=function(b){return a.handleEvent(b)});return a[$b]};function W(a){U.call(this);this.l=a;this.i={}}oa(W,U);var ac=[];function bc(a){ra(a.i,function(b,c){this.i.hasOwnProperty(c)&&Xb(b)},a);a.i={}}W.prototype.j=function(){W.o.j.call(this);bc(this)};W.prototype.handleEvent=function(){throw Error("EventHandler.handleEvent not implemented");};function cc(a){U.call(this);this.i=a||document.body;this.l=new W(this);a=na(yb,this.l);this.h?a():(this.g||(this.g=[]),this.g.push(a));a=this.l;var b=this.i,c=this.u,d="click";Array.isArray(d)||(d&&(ac[0]=d.toString()),d=ac);for(var e=0;e<d.length;e++){var f=Pb(b,d[e],c||a.handleEvent,!1,a.l||a);if(!f)break;a.i[f.key]=f}}h(cc,U);
cc.prototype.u=function(a){if(!(!(zb?0==a.i.button:"click"==a.type||a.i.button&Fb[0])||Sa&&a.ctrlKey||a.defaultPrevented))for(var b=a.target;b&&b!=this.i;){if(b.tagName&&"a"==b.tagName.toLowerCase()){var c=b.getAttribute("href")||b.getAttributeNS("http://www.w3.org/1999/xlink","href"),d=c;try{var e=rb(c)[3];var f;if(f="www.google.com"===pb(e,!0)){var g=rb(c)[5];f="/url"===pb(g,!0)}if(f){var k=sb(c,"q");d=k?k:sb(c,"url")}}catch(C){a:{var t=C;if(null!=t){var n=t.A;if(null!=n){C=n;break a}}if(t instanceof
TypeError){var R=n=new kb;R.l=B(t);hb(R);n.g="__noinit__";n.g=t;n.i(new TypeError(n));t=n}else R=n=new E,R.l=B(t),hb(R),n.g="__noinit__",n.g=t,n.i(Error(n)),t=n;C=t}if(!(C instanceof ib))throw C.h;}d=null!=d?d:"";if(c!=d){e=void 0;b={target:"_blank",noreferrer:!0};c=window;d instanceof q?f=d:(f="undefined"!=typeof d.href?d.href:String(d),f instanceof q||(f="object"==typeof f&&f.h?f.g():String(f),Fa.test(f)?f=new q(f,r):(f=String(f),f=f.replace(/(%0A|%0D)/g,""),f=(g=f.match(Ea))&&Da.test(g[1])?new q(f,
r):null)),f=f||Ha);d=b.target||d.target;g=[];for(e in b)switch(e){case "width":case "height":case "top":case "left":g.push(e+"="+b[e]);break;case "target":case "noopener":case "noreferrer":break;default:g.push(e+"="+(b[e]?1:0))}e=g.join(",");if((w("iPhone")&&!w("iPod")&&!w("iPad")||w("iPad")||w("iPod"))&&c.navigator&&c.navigator.standalone&&d&&"_self"!=d)e="A",g=document,e=String(e),"application/xhtml+xml"===g.contentType&&(e=e.toLowerCase()),g=e=g.createElement(e),f=f instanceof q?f:Ga(f),g.href=
u(f),e.setAttribute("target",d),b.noreferrer&&e.setAttribute("rel","noreferrer"),b=document.createEvent("MouseEvent"),b.initMouseEvent("click",!0,!0,c,1),e.dispatchEvent(b);else if(b.noreferrer){if(c=La("",c,d,e),b=u(f),c){Pa&&-1!=b.indexOf(";")&&(b="'"+b.replace(/'/g,"%27")+"'");c.opener=null;Ba.test(b)&&(-1!=b.indexOf("&")&&(b=b.replace(va,"&amp;")),-1!=b.indexOf("<")&&(b=b.replace(wa,"&lt;")),-1!=b.indexOf(">")&&(b=b.replace(xa,"&gt;")),-1!=b.indexOf('"')&&(b=b.replace(ya,"&quot;")),-1!=b.indexOf("'")&&
(b=b.replace(za,"&#39;")),-1!=b.indexOf("\x00")&&(b=b.replace(Aa,"&#0;")));b='<meta name="referrer" content="no-referrer"><meta http-equiv="refresh" content="0; url='+b+'">';if(void 0===m)if(d=null,(e=l.trustedTypes)&&e.createPolicy){try{d=e.createPolicy("goog#html",{createHTML:pa,createScript:pa,createScriptURL:pa})}catch(C){l.console&&l.console.error(C.message)}m=d}else m=d;b=(d=m)?d.createHTML(b):b;b=new x(b,null,Ka);(c=c.document)&&c.write&&(c.write(b instanceof x&&b.constructor===x?b.i:"type_error:SafeHtml"),
c.close())}}else(c=La(f,c,d,e))&&b.noopener&&(c.opener=null);a.h();break}}b=b.parentNode}};function dc(a){new cc(a)}var X=["DOCS_installLinkReferrerSanitizer"],Y=l;X[0]in Y||"undefined"==typeof Y.execScript||Y.execScript("var "+X[0]);for(var Z;X.length&&(Z=X.shift());)X.length||void 0===dc?Y[Z]&&Y[Z]!==Object.prototype[Z]?Y=Y[Z]:Y=Y[Z]={}:Y[Z]=dc;}).call(this);
</script><script type="text/javascript" nonce="">DOCS_installLinkReferrerSanitizer();</script></body></html>